diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 39dd579aa3732..2105e6de2cb64 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -26,6 +26,7 @@ updates: - "prost*" opentelemetry: patterns: + - "opentelemetry" - "opentelemetry*" - "tracing-opentelemetry" mysql: diff --git a/.github/workflows/cherry-pick-to-release-branch.yml b/.github/workflows/cherry-pick-to-release-branch.yml index 3406f7a43de11..ef56410c29c83 100644 --- a/.github/workflows/cherry-pick-to-release-branch.yml +++ b/.github/workflows/cherry-pick-to-release-branch.yml @@ -3,11 +3,11 @@ on: pull_request: branches: - main - types: ["closed"] + types: ["closed", "labeled"] jobs: - release_pull_request_1_5: - if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.5') && github.event.pull_request.merged == true" + release_pull_request_1_7_standalone: + if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7-standalone') && github.event.pull_request.merged == true" runs-on: ubuntu-latest name: release_pull_request steps: @@ -16,9 +16,25 @@ jobs: - name: Create PR to branch uses: risingwavelabs/github-action-cherry-pick@master with: - pr_branch: 'release-1.5' + pr_branch: 'release-1.7.0-standalone' pr_labels: 'cherry-pick' - pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.5', github.event.number) }} + pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.7.0-standalone', github.event.number) }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + release_pull_request_1_7: + if: "contains(github.event.pull_request.labels.*.name, 'need-cherry-pick-release-1.7') && github.event.pull_request.merged == true" + runs-on: ubuntu-latest + name: release_pull_request + steps: + - name: checkout + uses: actions/checkout@v1 + - name: Create PR to branch + uses: risingwavelabs/github-action-cherry-pick@master + with: + pr_branch: 'release-1.7' + pr_labels: 'cherry-pick' + pr_body: ${{ format('Cherry picking \#{0} onto branch release-1.7', github.event.number) }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.licenserc.yaml b/.licenserc.yaml index 137943a1f6dd0..93449f89340cb 100644 --- a/.licenserc.yaml +++ b/.licenserc.yaml @@ -17,6 +17,7 @@ header: - "**/*.d.ts" - "src/sqlparser/**/*.rs" - "java/connector-node/risingwave-source-cdc/src/main/java/com/risingwave/connector/cdc/debezium/internal/*.java" + - "java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/**/*.java" - "src/meta/model_v2/migration/**/*.rs" - "lints/ui/**" diff --git a/.typos.toml b/.typos.toml index 339ad5c42b703..b19d0a08c541d 100644 --- a/.typos.toml +++ b/.typos.toml @@ -20,7 +20,8 @@ extend-exclude = [ "scripts", "src/frontend/planner_test/tests/testdata", "src/tests/sqlsmith/tests/freeze", - "src/workspace-hack/Cargo.toml", + "Cargo.lock", + "**/Cargo.toml", "**/go.mod", "**/go.sum", ] diff --git a/Cargo.lock b/Cargo.lock index 01792a8b108d9..065078e7f7768 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -113,9 +113,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.5.0" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +checksum = "6e2e1ebcb11de5c03c67de28a7df593d32191b44939c482e97702baaaa6ab6a5" dependencies = [ "anstyle", "anstyle-parse", @@ -151,12 +151,12 @@ dependencies = [ [[package]] name = "anstyle-wincon" -version = "2.1.0" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -187,7 +187,7 @@ dependencies = [ "serde_json", "snap", "strum", - "strum_macros", + "strum_macros 0.25.3", "thiserror", "typed-builder 0.16.2", "uuid", @@ -212,7 +212,7 @@ dependencies = [ "serde", "serde_json", "strum", - "strum_macros", + "strum_macros 0.25.3", "thiserror", "typed-builder 0.18.0", "uuid", @@ -238,9 +238,9 @@ checksum = "7d5a26814d8dcb93b0e5a0ff3c6d80a8843bafb21b39e8e18a6f05471870e110" [[package]] name = "arc-swap" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bddcadddf5e9015d310179a59bb28c4d4b9920ad0f11e8e14dbadf654890c9a6" +checksum = "7b3d0060af21e8d11a926981cc00c6c1541aa91dd64b9f881985c3da1094425f" [[package]] name = "arrayref" @@ -293,14 +293,14 @@ dependencies = [ [[package]] name = "arrow-arith" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34ccd45e217ffa6e53bbb0080990e77113bdd4e91ddb84e97b77649810bcf1a7" +checksum = "753abd0a5290c1bcade7c6623a556f7d1659c5f4148b140b5b63ce7bd1a45705" dependencies = [ - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-data 49.0.0", - "arrow-schema 49.0.0", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-data 50.0.0", + "arrow-schema 50.0.0", "chrono", "half 2.3.1", "num", @@ -325,14 +325,14 @@ dependencies = [ [[package]] name = "arrow-array" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bda9acea48b25123c08340f3a8ac361aa0f74469bb36f5ee9acf923fce23e9d" +checksum = "d390feeb7f21b78ec997a4081a025baef1e2e0d6069e181939b61864c9779609" dependencies = [ "ahash 0.8.6", - "arrow-buffer 49.0.0", - "arrow-data 49.0.0", - "arrow-schema 49.0.0", + "arrow-buffer 50.0.0", + "arrow-data 50.0.0", + "arrow-schema 50.0.0", "chrono", "half 2.3.1", "hashbrown 0.14.0", @@ -352,9 +352,9 @@ dependencies = [ [[package]] name = "arrow-buffer" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01a0fc21915b00fc6c2667b069c1b64bdd920982f426079bc4a7cab86822886c" +checksum = "69615b061701bcdffbc62756bc7e85c827d5290b472b580c972ebbbf690f5aa4" dependencies = [ "bytes", "half 2.3.1", @@ -381,16 +381,16 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5dc0368ed618d509636c1e3cc20db1281148190a78f43519487b2daf07b63b4a" +checksum = "e448e5dd2f4113bf5b74a1f26531708f5edcacc77335b7066f9398f4bcf4cdef" dependencies = [ - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-data 49.0.0", - "arrow-schema 49.0.0", - "arrow-select 49.0.0", - "base64 0.21.4", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-data 50.0.0", + "arrow-schema 50.0.0", + "arrow-select 50.0.0", + "base64 0.21.7", "chrono", "half 2.3.1", "lexical-core", @@ -430,28 +430,28 @@ dependencies = [ [[package]] name = "arrow-data" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "907fafe280a3874474678c1858b9ca4cb7fd83fb8034ff5b6d6376205a08c634" +checksum = "67d644b91a162f3ad3135ce1184d0a31c28b816a581e08f29e8e9277a574c64e" dependencies = [ - "arrow-buffer 49.0.0", - "arrow-schema 49.0.0", + "arrow-buffer 50.0.0", + "arrow-schema 50.0.0", "half 2.3.1", "num", ] [[package]] name = "arrow-flight" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "624e0dcb6b5a7a06222bfd2be3f7e905ce849a6b714ec989f18cdba330c77d38" +checksum = "1d7f215461ad6346f2e4cc853e377d4e076d533e1ed78d327debe83023e3601f" dependencies = [ - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-cast 49.0.0", - "arrow-ipc 49.0.0", - "arrow-schema 49.0.0", - "base64 0.21.4", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-cast 50.0.0", + "arrow-ipc 50.0.0", + "arrow-schema 50.0.0", + "base64 0.21.7", "bytes", "futures", "paste", @@ -476,15 +476,15 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79a43d6808411886b8c7d4f6f7dd477029c1e77ffffffb7923555cc6579639cd" +checksum = "03dea5e79b48de6c2e04f03f62b0afea7105be7b77d134f6c5414868feefb80d" dependencies = [ - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-cast 49.0.0", - "arrow-data 49.0.0", - "arrow-schema 49.0.0", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-cast 50.0.0", + "arrow-data 50.0.0", + "arrow-schema 50.0.0", "flatbuffers", ] @@ -525,15 +525,15 @@ dependencies = [ [[package]] name = "arrow-ord" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b23b0e53c0db57c6749997fd343d4c0354c994be7eca67152dd2bdb9a3e1bb4" +checksum = "1ed9630979034077982d8e74a942b7ac228f33dd93a93b615b4d02ad60c260be" dependencies = [ - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-data 49.0.0", - "arrow-schema 49.0.0", - "arrow-select 49.0.0", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-data 50.0.0", + "arrow-schema 50.0.0", + "arrow-select 50.0.0", "half 2.3.1", "num", ] @@ -555,15 +555,15 @@ dependencies = [ [[package]] name = "arrow-row" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "361249898d2d6d4a6eeb7484be6ac74977e48da12a4dd81a708d620cc558117a" +checksum = "007035e17ae09c4e8993e4cb8b5b96edf0afb927cd38e2dff27189b274d83dcf" dependencies = [ "ahash 0.8.6", - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-data 49.0.0", - "arrow-schema 49.0.0", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-data 50.0.0", + "arrow-schema 50.0.0", "half 2.3.1", "hashbrown 0.14.0", ] @@ -579,9 +579,9 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09e28a5e781bf1b0f981333684ad13f5901f4cd2f20589eab7cf1797da8fc167" +checksum = "0ff3e9c01f7cd169379d269f926892d0e622a704960350d09d331be3ec9e0029" [[package]] name = "arrow-select" @@ -599,15 +599,15 @@ dependencies = [ [[package]] name = "arrow-select" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f6208466590960efc1d2a7172bc4ff18a67d6e25c529381d7f96ddaf0dc4036" +checksum = "1ce20973c1912de6514348e064829e50947e35977bb9d7fb637dc99ea9ffd78c" dependencies = [ "ahash 0.8.6", - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-data 49.0.0", - "arrow-schema 49.0.0", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-data 50.0.0", + "arrow-schema 50.0.0", "num", ] @@ -624,21 +624,51 @@ dependencies = [ "arrow-select 48.0.1", "num", "regex", - "regex-syntax 0.8.0", + "regex-syntax 0.8.2", ] [[package]] -name = "arrow-udf-wasm" +name = "arrow-udf-js" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "252b6355ad1e57eb6454b705c51652de55aa22eb018cdb95be0dbf62ee3ec78f" +dependencies = [ + "anyhow", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-schema 50.0.0", + "rquickjs", +] + +[[package]] +name = "arrow-udf-python" version = "0.1.0" -source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=f9a9e0d#f9a9e0d41d1a4ae26a6d90ac8aebf2e38a0c8a55" +source = "git+https://github.com/risingwavelabs/arrow-udf.git?rev=6c32f71#6c32f710b5948147f8214797fc334a4a3cadef0d" dependencies = [ "anyhow", - "arrow-array 49.0.0", - "arrow-ipc 49.0.0", - "arrow-schema 49.0.0", - "base64 0.21.4", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-ipc 50.0.0", + "arrow-schema 50.0.0", + "lazy_static", + "pyo3", + "pyo3-build-config", +] + +[[package]] +name = "arrow-udf-wasm" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ccaec984501c42dab31a0363e4c8b11e1dc2d870d7e3af6868b8ffbd704ce0a" +dependencies = [ + "anyhow", + "arrow-array 50.0.0", + "arrow-ipc 50.0.0", + "arrow-schema 50.0.0", + "base64 0.21.7", "genawaiter", "lazy_static", + "tempfile", "wasmtime", "wasmtime-wasi", ] @@ -670,19 +700,6 @@ dependencies = [ "futures-core", ] -[[package]] -name = "async-compat" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b48b4ff0c2026db683dea961cd8ea874737f56cffca86fa84415eaddc51c00d" -dependencies = [ - "futures-core", - "futures-io", - "once_cell", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-compression" version = "0.4.5" @@ -766,7 +783,7 @@ version = "0.33.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dbc1f1a75fd07f0f517322d103211f12d757658e91676def9a2e688774656c60" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bytes", "futures", "http 0.2.9", @@ -906,14 +923,13 @@ dependencies = [ [[package]] name = "auto_impl" -version = "1.1.0" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fee3da8ef1276b0bee5dd1c7258010d8fffd31801447323115a25560e1327b89" +checksum = "3c87f3f15e7794432337fc718554eaa4dc8f04c9677a950ffe366f20a162ae42" dependencies = [ - "proc-macro-error 1.0.4", "proc-macro2", "quote", - "syn 1.0.109", + "syn 2.0.48", ] [[package]] @@ -966,7 +982,7 @@ dependencies = [ "aws-smithy-types", "aws-types", "bytes", - "fastrand 2.0.0", + "fastrand 2.0.1", "http 0.2.9", "hyper", "time", @@ -976,9 +992,9 @@ dependencies = [ [[package]] name = "aws-credential-types" -version = "1.0.1" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1317e1a3514b103cf7d5828bbab3b4d30f56bd22d684f8568bc51b6cfbbb1c" +checksum = "4a7cb3510b95492bd9014b60e2e3bee3e48bc516e220316f8e6b60df18b47331" dependencies = [ "aws-smithy-async", "aws-smithy-runtime-api", @@ -1017,37 +1033,13 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", - "fastrand 2.0.0", + "fastrand 2.0.1", "http 0.2.9", "percent-encoding", "tracing", "uuid", ] -[[package]] -name = "aws-sdk-ec2" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88111ec31c6a3b9ed890f345b063bc2c76b49897e0c4601f6b1350c95d8d96d6" -dependencies = [ - "aws-credential-types", - "aws-http", - "aws-runtime", - "aws-smithy-async", - "aws-smithy-http", - "aws-smithy-json", - "aws-smithy-query", - "aws-smithy-runtime", - "aws-smithy-runtime-api", - "aws-smithy-types", - "aws-smithy-xml", - "aws-types", - "fastrand 2.0.0", - "http 0.2.9", - "regex", - "tracing", -] - [[package]] name = "aws-sdk-kinesis" version = "1.1.0" @@ -1125,9 +1117,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.0.1" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380adcc8134ad8bbdfeb2ace7626a869914ee266322965276cbc54066186d236" +checksum = "d222297ca90209dc62245f0a490355795f29de362eb5c19caea4f7f55fe69078" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1143,7 +1135,6 @@ dependencies = [ "once_cell", "p256 0.11.1", "percent-encoding", - "regex", "ring 0.17.5", "sha2", "subtle", @@ -1154,9 +1145,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.0.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fbfa248f7f966d73e325dbc85851a5500042b6d96e3c3b535a8527707f36fe4" +checksum = "2eac0bb78e9e2765699999a02d7bfb4e6ad8f13e0962ebb9f5202b1d8cd76006" dependencies = [ "futures-util", "pin-project-lite", @@ -1186,9 +1177,9 @@ dependencies = [ [[package]] name = "aws-smithy-eventstream" -version = "0.60.0" +version = "0.60.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c669e1e5fc0d79561bf7a122b118bd50c898758354fe2c53eb8f2d31507cbc3" +checksum = "682371561562d08ab437766903c6bc28f4f95d7ab2ecfb389bda7849dd98aefe" dependencies = [ "aws-smithy-types", "bytes", @@ -1197,9 +1188,9 @@ dependencies = [ [[package]] name = "aws-smithy-http" -version = "0.60.0" +version = "0.60.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1de8aee22f67de467b2e3d0dd0fb30859dc53f579a63bd5381766b987db644" +checksum = "365ca49744b2bda2f1e2dc03b856da3fa5a28ca5b0a41e41d7ff5305a8fae190" dependencies = [ "aws-smithy-eventstream", "aws-smithy-runtime-api", @@ -1246,7 +1237,7 @@ dependencies = [ "aws-smithy-runtime-api", "aws-smithy-types", "bytes", - "fastrand 2.0.0", + "fastrand 2.0.1", "http 0.2.9", "http-body", "hyper", @@ -1261,9 +1252,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime-api" -version = "1.0.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d27c3235d4972ed976b5c1a82286e7c4457f618f3c2ae6d4ae44f081dd24575" +checksum = "02ca2da7619517310bfead6d18abcdde90f1439224d887d608503cfacff46dff" dependencies = [ "aws-smithy-async", "aws-smithy-types", @@ -1276,9 +1267,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.1.1" +version = "1.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2aba8136605d14ac88f57dc3a693a9f8a4eab4a3f52bc03ff13746f0cd704e97" +checksum = "5d4bb944488536cd2fef43212d829bc7e9a8bfc4afa079d21170441e7be8d2d0" dependencies = [ "base64-simd", "bytes", @@ -1441,9 +1432,15 @@ checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" [[package]] name = "base64" -version = "0.21.4" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + +[[package]] +name = "base64" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ba43ea6f343b788c8764558649e08df62f86c6ef251fdaeb1ffd010a9ae50a2" +checksum = "9475866fec1451be56a3c2400fd081ff546538961565ccb5b7142cbd22bc7a51" [[package]] name = "base64-simd" @@ -1461,7 +1458,7 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c5b0a88aa36e9f095ee2e2b13fb8c5e4313e022783aedacc123328c0084916d" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", ] [[package]] @@ -1852,7 +1849,7 @@ checksum = "6ffc30dee200c20b4dcb80572226f42658e1d9c4b668656d7cc59c33d50e396e" dependencies = [ "cap-primitives", "cap-std", - "rustix 0.38.28", + "rustix 0.38.31", "smallvec", ] @@ -1868,7 +1865,7 @@ dependencies = [ "io-lifetimes 2.0.3", "ipnet", "maybe-owned", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.48.0", "winx", ] @@ -1892,7 +1889,7 @@ dependencies = [ "cap-primitives", "io-extras", "io-lifetimes 2.0.3", - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] @@ -1903,7 +1900,7 @@ checksum = "f8f52b3c8f4abfe3252fd0a071f3004aaa3b18936ec97bdbd8763ce03aff6247" dependencies = [ "cap-primitives", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "winx", ] @@ -2062,9 +2059,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.4.4" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1d7b8d5ec32af0fadc644bf1fd509a688c2103b185644bb1e29d164e0703136" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", "clap_derive", @@ -2072,21 +2069,21 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.4.4" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5179bb514e4d7c2051749d8fcefa2ed6d06a9f4e6d69faf3805f5d80b8cf8d56" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.0", ] [[package]] name = "clap_derive" -version = "4.4.2" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0862016ff20d69b84ef8247369fabf5c008a7417002411897d40ee1f4532b873" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -2096,9 +2093,9 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.5.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "clickhouse" @@ -2210,7 +2207,7 @@ checksum = "7c64043d6c7b7a4c58e39e7efccfdea7b93d885a795d0c054a69dbbf4dd52686" dependencies = [ "crossterm 0.27.0", "strum", - "strum_macros", + "strum_macros 0.25.3", "unicode-width", ] @@ -2376,18 +2373,18 @@ dependencies = [ [[package]] name = "cranelift-bforest" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c22542c0b95bd3302f7ed6839869c561f2324bac2fd5e7e99f5cfa65fdc8b92" +checksum = "d819feeda4c420a18f1e28236ca0ce1177b22bf7c8a44ddee92dfe40de15bcf0" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-codegen" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b3db903ef2e9c8a4de2ea6db5db052c7857282952f9df604aa55d169e6000d8" +checksum = "e9b8d03d5bdbca7e5f72b0e0a0f69933ed1f09e24be6c075aa6fe3f802b0cc0c" dependencies = [ "bumpalo", "cranelift-bforest", @@ -2406,33 +2403,33 @@ dependencies = [ [[package]] name = "cranelift-codegen-meta" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6590feb5a1d6438f974bf6a5ac4dddf69fca14e1f07f3265d880f69e61a94463" +checksum = "a3fd3664e38e51649b17dc30cfdd561273fe2f590dcd013fb75d9eabc6272dfb" dependencies = [ "cranelift-codegen-shared", ] [[package]] name = "cranelift-codegen-shared" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7239038c56fafe77fddc8788fc8533dd6c474dc5bdc5637216404f41ba807330" +checksum = "4b031ec5e605828975952622b5a77d49126f20ffe88d33719a0af66b23a0fc36" [[package]] name = "cranelift-control" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7dc9c595341404d381d27a3d950160856b35b402275f0c3990cd1ad683c8053" +checksum = "fada054d017cf2ed8f7ed2336e0517fc1b19e6825be1790de9eb00c94788362b" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44e3ee532fc4776c69bcedf7e62f9632cbb3f35776fa9a525cdade3195baa3f7" +checksum = "177b6f94ae8de6348eb45bf977c79ab9e3c40fc3ac8cb7ed8109560ea39bee7d" dependencies = [ "serde", "serde_derive", @@ -2440,9 +2437,9 @@ dependencies = [ [[package]] name = "cranelift-frontend" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a612c94d09e653662ec37681dc2d6fd2b9856e6df7147be0afc9aabb0abf19df" +checksum = "ebebd23a69a23e3ddea78e98ff3a2de222e88c8e045d81ef4a72f042e0d79dbd" dependencies = [ "cranelift-codegen", "log", @@ -2452,15 +2449,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85db9830abeb1170b7d29b536ffd55af1d4d26ac8a77570b5d1aca003bf225cc" +checksum = "1571bfc14df8966d12c6121b5325026591a4b4009e22fea0fe3765ab7cd33b96" [[package]] name = "cranelift-native" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "301ef0edafeaeda5771a5d2db64ac53e1818ae3111220a185677025fe91db4a1" +checksum = "35a69c37e0c10b46fe5527f2397ac821046efbf5f7ec112c8b84df25712f465b" dependencies = [ "cranelift-codegen", "libc", @@ -2469,9 +2466,9 @@ dependencies = [ [[package]] name = "cranelift-wasm" -version = "0.103.0" +version = "0.104.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "380f0abe8264e4570ac615fc31cef32a3b90a77f7eb97b08331f9dd357b1f500" +checksum = "9b3fef8bbceb8cb56d3f1778b0418d75c5cf12ec571a35fc01eb41abb0227a25" dependencies = [ "cranelift-codegen", "cranelift-entity", @@ -2509,9 +2506,9 @@ dependencies = [ [[package]] name = "crc32fast" -version = "1.3.2" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d" +checksum = "b3855a8a784b474f333699ef2bbca9db2c4a1f6d9088a90a2d25b1eb53111eaa" dependencies = [ "cfg-if", ] @@ -2733,16 +2730,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "ctor" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d2b3721e861707777e3195b0158f950ae6dc4a27e4d02ff9f67e3eb3de199e" -dependencies = [ - "quote", - "syn 2.0.48", -] - [[package]] name = "curve25519-dalek" version = "4.1.0" @@ -2855,7 +2842,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] @@ -2869,7 +2856,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 1.0.109", ] @@ -2883,7 +2870,7 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.10.0", "syn 2.0.48", ] @@ -3065,7 +3052,7 @@ dependencies = [ "datafusion-common", "sqlparser", "strum", - "strum_macros", + "strum_macros 0.25.3", ] [[package]] @@ -3083,7 +3070,7 @@ dependencies = [ "hashbrown 0.14.0", "itertools 0.11.0", "log", - "regex-syntax 0.8.0", + "regex-syntax 0.8.2", ] [[package]] @@ -3098,7 +3085,7 @@ dependencies = [ "arrow-buffer 48.0.1", "arrow-ord 48.0.1", "arrow-schema 48.0.1", - "base64 0.21.4", + "base64 0.21.7", "blake2", "blake3", "chrono", @@ -3212,6 +3199,7 @@ dependencies = [ name = "delta_btree_map" version = "1.7.0-alpha" dependencies = [ + "educe 0.5.7", "enum-as-inner", ] @@ -3591,9 +3579,9 @@ dependencies = [ [[package]] name = "either" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a26ae43d7bcc3b814de94796a5e736d4029efb0ee900c12e2d54c993ad1a1e07" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" dependencies = [ "serde", ] @@ -3688,18 +3676,18 @@ dependencies = [ [[package]] name = "enum-iterator" -version = "1.4.1" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7add3873b5dd076766ee79c8e406ad1a472c385476b9e38849f8eec24f1be689" +checksum = "600536cfe9e2da0820aa498e570f6b2b9223eec3ce2f835c8ae4861304fa4794" dependencies = [ "enum-iterator-derive", ] [[package]] name = "enum-iterator-derive" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eecf8589574ce9b895052fa12d69af7a233f99e6107f5cb8dd1044f2a17bfdcb" +checksum = "03cdc46ec28bd728e67540c528013c6a10eb69a02eb31078a1bda695438cbfb8" dependencies = [ "proc-macro2", "quote", @@ -3797,6 +3785,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "escape8259" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba4f4911e3666fcd7826997b4745c8224295a6f3072f1418c3067b97a67557ee" +dependencies = [ + "rustversion", +] + [[package]] name = "etcd-client" version = "0.12.1" @@ -3878,8 +3875,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "531e46835a22af56d1e3b66f04844bed63158bc094a628bec1d321d9b4c44bf2" dependencies = [ "bit-set", - "regex-automata 0.4.1", - "regex-syntax 0.8.0", + "regex-automata 0.4.5", + "regex-syntax 0.8.2", ] [[package]] @@ -3902,9 +3899,9 @@ dependencies = [ [[package]] name = "fastrand" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6999dc1837253364c2ebb0704ba97994bd874e8f195d665c50b7548f6ea92764" +checksum = "25cbce373ec4653f1a01a31e8a5e5ec0c622dc27ff9c4e6606eefef5cbbed4a5" [[package]] name = "fd-lock" @@ -3913,7 +3910,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b93f7a0db71c99f68398f80653ed05afb0b00e062e1a20c7ff849c4edfabbbcc" dependencies = [ "cfg-if", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -3996,6 +3993,12 @@ version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" +[[package]] +name = "fixedbitset" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b51ee430d5ff16df7998870eb0b4775383ac5bc450f5a2ed547394fe2d617131" + [[package]] name = "flagset" version = "0.4.3" @@ -4277,7 +4280,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "033b337d725b97690d86893f9de22b67b80dcc4e9ad815f348254c38119db8fb" dependencies = [ "io-lifetimes 2.0.3", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -4351,9 +4354,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" dependencies = [ "futures-core", "futures-sink", @@ -4361,9 +4364,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" [[package]] name = "futures-executor" @@ -4389,9 +4392,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" [[package]] name = "futures-lite" @@ -4410,9 +4413,9 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" dependencies = [ "proc-macro2", "quote", @@ -4421,15 +4424,15 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" [[package]] name = "futures-timer" @@ -4441,9 +4444,9 @@ dependencies = [ [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" dependencies = [ "futures-channel", "futures-core", @@ -4457,6 +4460,15 @@ dependencies = [ "slab", ] +[[package]] +name = "fuzzy-matcher" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54614a3312934d066701a80f20f15fa3b56d67ac7722b39eea5b4c9dd1d66c94" +dependencies = [ + "thread_local", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -4579,6 +4591,23 @@ dependencies = [ "stable_deref_trait", ] +[[package]] +name = "ginepro" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eedbff62a689be48f58f32571dbf3d60c4a73b39740141dfe7ac942536ea27f7" +dependencies = [ + "anyhow", + "async-trait", + "http 0.2.9", + "thiserror", + "tokio", + "tonic 0.10.2", + "tower", + "tracing", + "trust-dns-resolver", +] + [[package]] name = "glob" version = "0.3.1" @@ -4604,7 +4633,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1087f1fbd2dd3f58c17c7574ddd99cd61cbbbc2c4dc81114b8687209b196cb" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.7", "google-cloud-metadata", "google-cloud-token", "home", @@ -4659,9 +4688,9 @@ dependencies = [ [[package]] name = "google-cloud-pubsub" -version = "0.22.0" +version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72c96d7e0fd07ecbce85bc8420bb7a049bb70ea74036e59fbee1a51deaa79216" +checksum = "1da196da473976944d408a91213bafe078e7223e10694d3f8ed36b6e210fa130" dependencies = [ "async-channel", "async-stream", @@ -4726,9 +4755,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -4736,7 +4765,7 @@ dependencies = [ "futures-sink", "futures-util", "http 0.2.9", - "indexmap 1.9.3", + "indexmap 2.0.0", "slab", "tokio", "tokio-util", @@ -5051,18 +5080,18 @@ dependencies = [ [[package]] name = "icelake" version = "0.0.10" -source = "git+https://github.com/icelake-io/icelake?rev=32c0bbf242f5c47b1e743f10577012fe7436c770#32c0bbf242f5c47b1e743f10577012fe7436c770" +source = "git+https://github.com/icelake-io/icelake?rev=54fd72fbd1dd8c592f05eeeb79223c8a6a33c297#54fd72fbd1dd8c592f05eeeb79223c8a6a33c297" dependencies = [ "anyhow", "apache-avro 0.17.0", - "arrow-arith 49.0.0", - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-cast 49.0.0", - "arrow-ord 49.0.0", - "arrow-row 49.0.0", - "arrow-schema 49.0.0", - "arrow-select 49.0.0", + "arrow-arith 50.0.0", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-cast 50.0.0", + "arrow-ord 50.0.0", + "arrow-row 50.0.0", + "arrow-schema 50.0.0", + "arrow-select 50.0.0", "async-trait", "bitvec", "bytes", @@ -5079,7 +5108,7 @@ dependencies = [ "once_cell", "opendal", "ordered-float 3.9.1", - "parquet 49.0.0", + "parquet 50.0.0", "prometheus", "regex", "reqwest", @@ -5107,6 +5136,16 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" +[[package]] +name = "idna" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c" +dependencies = [ + "unicode-bidi", + "unicode-normalization", +] + [[package]] name = "idna" version = "0.5.0" @@ -5195,16 +5234,17 @@ dependencies = [ [[package]] name = "inquire" -version = "0.6.2" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33e7c1ddeb15c9abcbfef6029d8e29f69b52b6d6c891031b88ed91b5065803b" +checksum = "bd05e4e63529f3c9c5f5c668c398217f72756ffe48c85266b49692c55accd1f7" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.0", "crossterm 0.25.0", "dyn-clone", - "lazy_static", + "fuzzy-matcher", + "fxhash", "newline-converter", - "thiserror", + "once_cell", "unicode-segmentation", "unicode-width", ] @@ -5251,6 +5291,18 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a611371471e98973dbcab4e0ec66c31a10bc356eeb4d54a0e05eac8158fe38c" +[[package]] +name = "ipconfig" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b58db92f96b720de98181bbbe63c831e87005ab460c1bf306eb2622b4707997f" +dependencies = [ + "socket2 0.5.3", + "widestring", + "windows-sys 0.48.0", + "winreg", +] + [[package]] name = "ipnet" version = "2.8.0" @@ -5264,7 +5316,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" dependencies = [ "hermit-abi", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.48.0", ] @@ -5406,7 +5458,7 @@ version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6971da4d9c3aa03c3d8f3ff0f4155b534aad021292003895a469716b2a230378" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "pem 1.1.1", "ring 0.16.20", "serde", @@ -5420,7 +5472,7 @@ version = "9.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c7ea04a7c5c055c175f189b6dc6ba036fd62306b58c66c9f6389036c503a3f4" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "js-sys", "pem 3.0.2", "ring 0.17.5", @@ -5543,9 +5595,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.150" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libflate" @@ -5620,6 +5672,18 @@ dependencies = [ "threadpool", ] +[[package]] +name = "libtest-mimic" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0f4c6f44ecfd52e8b443f2ad18f2b996540135771561283c2352ce56a1c70b" +dependencies = [ + "clap", + "escape8259", + "termcolor", + "threadpool", +] + [[package]] name = "libz-sys" version = "1.1.12" @@ -5647,6 +5711,26 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" +[[package]] +name = "linkme" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b53ad6a33de58864705954edb5ad5d571a010f9e296865ed43dc72a5621b430" +dependencies = [ + "linkme-impl", +] + +[[package]] +name = "linkme-impl" +version = "0.3.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04e542a18c94a9b6fcc7adb090fa3ba6b79ee220a16404f325672729f32a66ff" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", +] + [[package]] name = "linux-raw-sys" version = "0.1.4" @@ -5710,9 +5794,9 @@ dependencies = [ [[package]] name = "lru" version = "0.7.6" -source = "git+https://github.com/risingwavelabs/lru-rs.git?rev=cb2d7c7#cb2d7c7149a7f320c5aa73c15a6ec9f46ed5513f" +source = "git+https://github.com/risingwavelabs/lru-rs.git?rev=95f347b#95f347b5ca3c947335a51adf40fa86178ef4d60d" dependencies = [ - "hashbrown 0.12.3", + "hashbrown 0.14.0", ] [[package]] @@ -5733,6 +5817,15 @@ dependencies = [ "hashbrown 0.14.0", ] +[[package]] +name = "lru-cache" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c" +dependencies = [ + "linked-hash-map", +] + [[package]] name = "lz4" version = "1.24.0" @@ -5993,9 +6086,9 @@ checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" [[package]] name = "memchr" -version = "2.6.3" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f232d6ef707e1956a43342693d2a31e72989554d58299d7a88738cc95b0d35c" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "memcomparable" @@ -6015,7 +6108,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2cffa4ad52c6f791f4f8b15f0c05f9824b2ced1160e88cc393d64fff9a8ac64" dependencies = [ - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] @@ -6069,9 +6162,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "log", @@ -6144,9 +6237,9 @@ checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" [[package]] name = "multimap" -version = "0.9.0" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70db9248a93dc36a36d9a47898caa007a32755c7ad140ec64eeeb50d5a730631" +checksum = "defc4c55412d89136f966bbb339008b474350e5e6e78d2714439c386b3137a03" dependencies = [ "serde", ] @@ -6215,7 +6308,7 @@ version = "0.31.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06f19e4cfa0ab5a76b627cec2d81331c49b034988eaf302c3bafeada684eadef" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bigdecimal 0.4.2", "bindgen", "bitflags 2.4.0", @@ -6275,9 +6368,9 @@ dependencies = [ [[package]] name = "newline-converter" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f71d09d5c87634207f894c6b31b6a2b2c64ea3bdcf71bd5599fdbbe1600c00f" +checksum = "47b6b097ecb1cbfed438542d16e84fd7ad9b0c76c8a65b7f9039212a3d14dc7f" dependencies = [ "unicode-segmentation", ] @@ -6315,6 +6408,18 @@ dependencies = [ "libc", ] +[[package]] +name = "nix" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab2156c4fce2f8df6c499cc1c763e4394b7482525bf2a9701c9d79d215f519e4" +dependencies = [ + "bitflags 2.4.0", + "cfg-if", + "cfg_aliases", + "libc", +] + [[package]] name = "nkeys" version = "0.3.2" @@ -6576,7 +6681,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f930c88a43b1c3f6e776dfe495b4afab89882dbc81530c632db2ed65451ebcb4" dependencies = [ "async-trait", - "base64 0.21.4", + "base64 0.21.7", "bytes", "chrono", "futures", @@ -6612,15 +6717,14 @@ checksum = "0ab1bc2a289d34bd04a330323ac98a1b4bc82c9d9fcb1e66b63caa84da26b575" [[package]] name = "opendal" -version = "0.44.0" +version = "0.44.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c32736a48ef08a5d2212864e2295c8e54f4d6b352b7f49aa0c29a12fc410ff66" +checksum = "4af824652d4d2ffabf606d337a071677ae621b05622adf35df9562f69d9b4498" dependencies = [ "anyhow", - "async-compat", "async-trait", "backon", - "base64 0.21.4", + "base64 0.21.7", "bytes", "chrono", "flagset", @@ -6630,9 +6734,7 @@ dependencies = [ "log", "md-5", "once_cell", - "parking_lot 0.12.1", "percent-encoding", - "pin-project", "prometheus", "quick-xml 0.30.0", "reqsign", @@ -6732,26 +6834,32 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.20.0" +version = "0.21.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9591d937bc0e6d2feb6f71a559540ab300ea49955229c347a517a28d27784c54" +checksum = "1e32339a5dc40459130b3bd269e9892439f55b33e772d2a9d402a789baaf4e8a" dependencies = [ - "opentelemetry_api", - "opentelemetry_sdk", + "futures-core", + "futures-sink", + "indexmap 2.0.0", + "js-sys", + "once_cell", + "pin-project-lite", + "thiserror", + "urlencoding", ] [[package]] name = "opentelemetry-otlp" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e5e5a5c4135864099f3faafbe939eb4d7f9b80ebf68a8448da961b32a7c1275" +checksum = "f24cda83b20ed2433c68241f918d0f6fdec8b1d43b7a9590ab4420c5095ca930" dependencies = [ "async-trait", "futures-core", "http 0.2.9", + "opentelemetry", "opentelemetry-proto", "opentelemetry-semantic-conventions", - "opentelemetry_api", "opentelemetry_sdk", "prost 0.11.9", "thiserror", @@ -6761,11 +6869,11 @@ dependencies = [ [[package]] name = "opentelemetry-proto" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e3f814aa9f8c905d0ee4bde026afd3b2577a97c10e1699912e3e44f0c4cbeb" +checksum = "a2e155ce5cc812ea3d1dffbd1539aed653de4bf4882d60e6e04dcf0901d674e1" dependencies = [ - "opentelemetry_api", + "opentelemetry", "opentelemetry_sdk", "prost 0.11.9", "tonic 0.9.2", @@ -6773,47 +6881,30 @@ dependencies = [ [[package]] name = "opentelemetry-semantic-conventions" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73c9f9340ad135068800e7f1b24e9e09ed9e7143f5bf8518ded3d3ec69789269" +checksum = "f5774f1ef1f982ef2a447f6ee04ec383981a3ab99c8e77a1a7b30182e65bbc84" dependencies = [ "opentelemetry", ] [[package]] -name = "opentelemetry_api" -version = "0.20.0" +name = "opentelemetry_sdk" +version = "0.21.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a81f725323db1b1206ca3da8bb19874bbd3f57c3bcd59471bfb04525b265b9b" +checksum = "968ba3f2ca03e90e5187f5e4f46c791ef7f2c163ae87789c8ce5f5ca3b7b7de5" dependencies = [ + "async-trait", + "crossbeam-channel", "futures-channel", + "futures-executor", "futures-util", - "indexmap 1.9.3", - "js-sys", + "glob", "once_cell", - "pin-project-lite", - "thiserror", - "urlencoding", -] - -[[package]] -name = "opentelemetry_sdk" -version = "0.20.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8e705a0612d48139799fcbaba0d4a90f06277153e43dd2bdc16c6f0edd8026" -dependencies = [ - "async-trait", - "crossbeam-channel", - "futures-channel", - "futures-executor", - "futures-util", - "once_cell", - "opentelemetry_api", - "ordered-float 3.9.1", + "opentelemetry", + "ordered-float 4.1.1", "percent-encoding", "rand", - "regex", - "serde_json", "thiserror", "tokio", "tokio-stream", @@ -6837,6 +6928,15 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ordered-float" +version = "4.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "536900a8093134cf9ccf00a27deb3532421099e958d9dd431135d0c7543ca1e8" +dependencies = [ + "num-traits", +] + [[package]] name = "ordered-multimap" version = "0.7.1" @@ -7031,7 +7131,7 @@ dependencies = [ "arrow-ipc 48.0.1", "arrow-schema 48.0.1", "arrow-select 48.0.1", - "base64 0.21.4", + "base64 0.21.7", "brotli", "bytes", "chrono", @@ -7053,24 +7153,25 @@ dependencies = [ [[package]] name = "parquet" -version = "49.0.0" +version = "50.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af88740a842787da39b3d69ce5fbf6fce97d20211d3b299fee0a0da6430c74d4" +checksum = "547b92ebf0c1177e3892f44c8f79757ee62e678d564a9834189725f2c5b7a750" dependencies = [ "ahash 0.8.6", - "arrow-array 49.0.0", - "arrow-buffer 49.0.0", - "arrow-cast 49.0.0", - "arrow-data 49.0.0", - "arrow-ipc 49.0.0", - "arrow-schema 49.0.0", - "arrow-select 49.0.0", - "base64 0.21.4", + "arrow-array 50.0.0", + "arrow-buffer 50.0.0", + "arrow-cast 50.0.0", + "arrow-data 50.0.0", + "arrow-ipc 50.0.0", + "arrow-schema 50.0.0", + "arrow-select 50.0.0", + "base64 0.21.7", "brotli", "bytes", "chrono", "flate2", "futures", + "half 2.3.1", "hashbrown 0.14.0", "lz4_flex", "num", @@ -7086,26 +7187,25 @@ dependencies = [ [[package]] name = "parse-display" -version = "0.8.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c6509d08722b53e8dafe97f2027b22ccbe3a5db83cb352931e9716b0aa44bc5c" +checksum = "06af5f9333eb47bd9ba8462d612e37a8328a5cb80b13f0af4de4c3b89f52dee5" dependencies = [ - "once_cell", "parse-display-derive", "regex", + "regex-syntax 0.8.2", ] [[package]] name = "parse-display-derive" -version = "0.8.2" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68517892c8daf78da08c0db777fcc17e07f2f63ef70041718f8a7630ad84f341" +checksum = "dc9252f259500ee570c75adcc4e317fa6f57a1e47747d622e0bf838002a7b790" dependencies = [ - "once_cell", "proc-macro2", "quote", "regex", - "regex-syntax 0.7.5", + "regex-syntax 0.8.2", "structmeta", "syn 2.0.48", ] @@ -7149,7 +7249,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "serde", ] @@ -7186,7 +7286,7 @@ version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3163d2912b7c3b52d651a055f2c7eec9ba5cd22d26ef75b8dd3a59980b185923" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "serde", ] @@ -7211,7 +7311,7 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ - "fixedbitset", + "fixedbitset 0.4.2", "indexmap 2.0.0", ] @@ -7236,11 +7336,16 @@ dependencies = [ "bytes", "futures", "itertools 0.12.0", + "jsonwebtoken 9.2.0", "madsim-tokio", "openssl", "panic-message", + "parking_lot 0.12.1", + "reqwest", "risingwave_common", "risingwave_sqlparser", + "serde", + "serde_json", "tempfile", "thiserror", "thiserror-ext", @@ -7446,7 +7551,7 @@ version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49b6c5ef183cd3ab4ba005f1ca64c21e8bd97ce4699cfea9e8d9a2c4958ca520" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "byteorder", "bytes", "fallible-iterator 0.2.0", @@ -7471,6 +7576,7 @@ dependencies = [ "postgres-protocol", "serde", "serde_json", + "uuid", ] [[package]] @@ -7674,9 +7780,9 @@ checksum = "dc375e1527247fe1a97d8b7156678dfe7c1af2fc075c9a4db3690ecd2a148068" [[package]] name = "proc-macro2" -version = "1.0.76" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95fc56cda0b5c3325f5fbbd7ff9fda9e02bb00bb3dac51252d2f1bfa1cb8cc8c" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] @@ -7704,7 +7810,7 @@ dependencies = [ "hex", "lazy_static", "procfs-core", - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] @@ -7798,7 +7904,7 @@ checksum = "8bdf592881d821b83d471f8af290226c8d51402259e9bb5be7f9f8bdebbb11ac" dependencies = [ "bytes", "heck 0.4.1", - "itertools 0.10.5", + "itertools 0.11.0", "log", "multimap 0.8.3", "once_cell", @@ -7832,7 +7938,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "265baba7fabd416cf5078179f7d2cbeca4ce7a9041111900675ea7c4cb8a4c32" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.11.0", "proc-macro2", "quote", "syn 2.0.48", @@ -7849,9 +7955,9 @@ dependencies = [ [[package]] name = "prost-reflect" -version = "0.12.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "057237efdb71cf4b3f9396302a3d6599a92fa94063ba537b66130980ea9909f3" +checksum = "9ae9372e3227f3685376a0836e5c248611eafc95a0be900d44bc6cdf225b700f" dependencies = [ "once_cell", "prost 0.12.1", @@ -7961,6 +8067,7 @@ dependencies = [ "futures-io", "futures-timer", "log", + "lz4", "native-tls", "nom", "oauth2", @@ -7979,6 +8086,70 @@ dependencies = [ "tracing", "url", "uuid", + "zstd 0.12.4", +] + +[[package]] +name = "pyo3" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53bdbb96d49157e65d45cc287af5f32ffadd5f4761438b527b055fb0d4bb8233" +dependencies = [ + "cfg-if", + "indoc", + "libc", + "memoffset", + "parking_lot 0.12.1", + "portable-atomic", + "pyo3-build-config", + "pyo3-ffi", + "pyo3-macros", + "unindent", +] + +[[package]] +name = "pyo3-build-config" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deaa5745de3f5231ce10517a1f5dd97d53e5a2fd77aa6b5842292085831d48d7" +dependencies = [ + "once_cell", + "target-lexicon", +] + +[[package]] +name = "pyo3-ffi" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62b42531d03e08d4ef1f6e85a2ed422eb678b8cd62b762e53891c05faf0d4afa" +dependencies = [ + "libc", + "pyo3-build-config", +] + +[[package]] +name = "pyo3-macros" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7305c720fa01b8055ec95e484a6eca7a83c841267f0dd5280f0c8b8551d2c158" +dependencies = [ + "proc-macro2", + "pyo3-macros-backend", + "quote", + "syn 2.0.48", +] + +[[package]] +name = "pyo3-macros-backend" +version = "0.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c7e9b68bb9c3149c5b0cade5d07f953d6d125eb4337723c4ccdb665f1f96185" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "pyo3-build-config", + "quote", + "syn 2.0.48", ] [[package]] @@ -8124,9 +8295,9 @@ dependencies = [ [[package]] name = "rdkafka-sys" -version = "4.6.0+2.2.0" +version = "4.7.0+2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad63c279fca41a27c231c450a2d2ad18288032e9cbb159ad16c9d96eba35aaaf" +checksum = "55e0d2f9ba6253f6ec72385e453294f8618e9e15c2c6aba2a5c01ccf9622d615" dependencies = [ "cmake", "libc", @@ -8239,14 +8410,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.0" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d119d7c7ca818f8a53c300863d4f87566aac09943aef5b355bb83969dae75d87" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.1", - "regex-syntax 0.8.0", + "regex-automata 0.4.5", + "regex-syntax 0.8.2", ] [[package]] @@ -8260,13 +8431,13 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.1" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465c6fc0621e4abc4187a2bda0937bfd4f722c2730b29562e19689ea796c9a4b" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.0", + "regex-syntax 0.8.2", ] [[package]] @@ -8283,15 +8454,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - -[[package]] -name = "regex-syntax" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3cbb081b9784b07cceb8824c8583f86db4814d172ab043f3c23f7dc600bf83d" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rend" @@ -8310,7 +8475,7 @@ checksum = "dce87f66ba6c6acef277a729f989a0eca946cb9ce6a15bcc036bda0f72d4b9fd" dependencies = [ "anyhow", "async-trait", - "base64 0.21.4", + "base64 0.21.7", "chrono", "form_urlencoded", "getrandom", @@ -8340,7 +8505,7 @@ version = "0.11.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e9ad3fe7488d7e34558a2033d45a0c90b72d97b4f80705666fea71472e2e6a1" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "bytes", "encoding_rs", "futures-core", @@ -8379,6 +8544,16 @@ dependencies = [ "winreg", ] +[[package]] +name = "resolv-conf" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52e44394d2086d010551b14b53b1f24e31647570cd1deb0379e2c21b329aba00" +dependencies = [ + "hostname", + "quick-error", +] + [[package]] name = "retain_mut" version = "0.1.7" @@ -8467,6 +8642,7 @@ dependencies = [ "serde_with", "serde_yaml", "tempfile", + "thiserror-ext", "tracing", "tracing-subscriber", "workspace-hack", @@ -8525,6 +8701,8 @@ name = "risingwave_batch" version = "1.7.0-alpha" dependencies = [ "anyhow", + "arrow-array 50.0.0", + "arrow-schema 50.0.0", "assert_matches", "async-recursion", "async-trait", @@ -8535,6 +8713,7 @@ dependencies = [ "futures-util", "hashbrown 0.14.0", "hytra", + "icelake", "itertools 0.12.0", "madsim-tokio", "madsim-tonic", @@ -8545,13 +8724,14 @@ dependencies = [ "rand", "risingwave_common", "risingwave_connector", + "risingwave_dml", "risingwave_expr", "risingwave_expr_impl", "risingwave_hummock_sdk", "risingwave_pb", "risingwave_rpc_client", - "risingwave_source", "risingwave_storage", + "rw_futures_util", "scopeguard", "serde_json", "task_stats_alloc", @@ -8570,6 +8750,7 @@ dependencies = [ name = "risingwave_bench" version = "1.7.0-alpha" dependencies = [ + "anyhow", "async-trait", "aws-config", "aws-sdk-s3", @@ -8580,19 +8761,25 @@ dependencies = [ "bytesize", "clap", "futures", + "futures-async-stream", "hdrhistogram", "itertools 0.12.0", "libc", "madsim-tokio", - "nix 0.27.1", + "nix 0.28.0", "opentelemetry", "parking_lot 0.12.1", "prometheus", "rand", "risingwave_common", + "risingwave_connector", + "risingwave_pb", "risingwave_rt", "risingwave_storage", + "risingwave_stream", "serde", + "serde_yaml", + "thiserror-ext", "tokio-stream", "toml 0.8.2", "tracing", @@ -8630,6 +8817,7 @@ dependencies = [ "console", "const-str", "expect-test", + "home", "madsim-tokio", "prometheus", "risingwave_cmd", @@ -8637,15 +8825,17 @@ dependencies = [ "risingwave_compactor", "risingwave_compute", "risingwave_ctl", + "risingwave_expr", "risingwave_expr_impl", "risingwave_frontend", "risingwave_meta_node", "risingwave_rt", "shell-words", "strum", - "strum_macros", + "strum_macros 0.26.1", "task_stats_alloc", "tempfile", + "thiserror-ext", "tikv-jemallocator", "tracing", "vergen", @@ -8660,13 +8850,13 @@ dependencies = [ "anyhow", "arc-swap", "arrow-array 48.0.1", - "arrow-array 49.0.0", + "arrow-array 50.0.0", "arrow-buffer 48.0.1", - "arrow-buffer 49.0.0", + "arrow-buffer 50.0.0", "arrow-cast 48.0.1", - "arrow-cast 49.0.0", + "arrow-cast 50.0.0", "arrow-schema 48.0.1", - "arrow-schema 49.0.0", + "arrow-schema 50.0.0", "async-trait", "auto_enums", "auto_impl", @@ -8687,7 +8877,7 @@ dependencies = [ "enumflags2", "ethnum", "expect-test", - "fixedbitset", + "fixedbitset 0.5.0", "fs-err", "futures", "governor", @@ -8711,6 +8901,7 @@ dependencies = [ "num-traits", "number_prefix", "opentelemetry", + "opentelemetry_sdk", "parking_lot 0.12.1", "parse-display", "paste", @@ -8730,6 +8921,7 @@ dependencies = [ "risingwave_pb", "rust_decimal", "rusty-fork", + "rw_futures_util", "ryu", "serde", "serde_bytes", @@ -8740,7 +8932,7 @@ dependencies = [ "speedate", "static_assertions", "strum", - "strum_macros", + "strum_macros 0.26.1", "sysinfo", "tempfile", "thiserror", @@ -8779,6 +8971,7 @@ name = "risingwave_common_proc_macro" version = "1.7.0-alpha" dependencies = [ "bae", + "itertools 0.12.0", "proc-macro-error 1.0.4", "proc-macro2", "quote", @@ -8880,11 +9073,11 @@ dependencies = [ "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_connector", + "risingwave_dml", "risingwave_hummock_sdk", "risingwave_jni_core", "risingwave_pb", "risingwave_rpc_client", - "risingwave_source", "risingwave_storage", "risingwave_stream", "serde", @@ -8904,8 +9097,11 @@ version = "1.7.0-alpha" dependencies = [ "anyhow", "apache-avro 0.16.0", - "arrow-array 49.0.0", - "arrow-schema 49.0.0", + "arrow-array 50.0.0", + "arrow-row 50.0.0", + "arrow-schema 50.0.0", + "arrow-select 50.0.0", + "assert_matches", "async-nats", "async-trait", "auto_enums", @@ -8919,7 +9115,7 @@ dependencies = [ "aws-smithy-types", "aws-smithy-types-convert", "aws-types", - "base64 0.21.4", + "base64 0.22.0", "byteorder", "bytes", "chrono", @@ -8946,6 +9142,7 @@ dependencies = [ "jsonschema-transpiler", "madsim-rdkafka", "madsim-tokio", + "madsim-tonic", "maplit", "moka", "mysql_async", @@ -8974,6 +9171,7 @@ dependencies = [ "risingwave_pb", "risingwave_rpc_client", "rust_decimal", + "rw_futures_util", "serde", "serde_derive", "serde_json", @@ -8981,21 +9179,22 @@ dependencies = [ "serde_yaml", "simd-json", "strum", - "strum_macros", + "strum_macros 0.26.1", "syn 1.0.109", "tempfile", "thiserror", + "thiserror-ext", "time", "tokio-postgres", "tokio-retry", "tokio-stream", "tokio-util", - "tonic 0.10.2", "tracing", "tracing-subscriber", "tracing-test", "url", "urlencoding", + "uuid", "walkdir", "with_options", "workspace-hack", @@ -9012,10 +9211,12 @@ dependencies = [ "clap", "comfy-table", "futures", + "hex", "inquire", "itertools 0.12.0", "madsim-etcd-client", "madsim-tokio", + "prost 0.12.1", "regex", "risingwave_common", "risingwave_connector", @@ -9037,6 +9238,30 @@ dependencies = [ "workspace-hack", ] +[[package]] +name = "risingwave_dml" +version = "1.7.0-alpha" +dependencies = [ + "assert_matches", + "criterion", + "futures", + "futures-async-stream", + "itertools 0.12.0", + "madsim-tokio", + "parking_lot 0.12.1", + "paste", + "rand", + "risingwave_common", + "risingwave_connector", + "risingwave_pb", + "rw_futures_util", + "tempfile", + "thiserror", + "thiserror-ext", + "tracing", + "workspace-hack", +] + [[package]] name = "risingwave_e2e_extended_mode_test" version = "1.7.0-alpha" @@ -9056,6 +9281,7 @@ dependencies = [ name = "risingwave_error" version = "1.7.0-alpha" dependencies = [ + "anyhow", "bincode 1.3.3", "bytes", "easy-ext", @@ -9072,31 +9298,35 @@ name = "risingwave_expr" version = "1.7.0-alpha" dependencies = [ "anyhow", - "arrow-array 49.0.0", - "arrow-schema 49.0.0", + "arrow-array 50.0.0", + "arrow-schema 50.0.0", + "arrow-udf-js", + "arrow-udf-python", "arrow-udf-wasm", "async-trait", "auto_impl", "await-tree", "cfg-or-panic", "chrono", - "ctor", "downcast-rs", "easy-ext", + "educe 0.5.7", "either", "enum-as-inner", "expect-test", "futures-async-stream", "futures-util", "itertools 0.12.0", + "linkme", "madsim-tokio", + "md5", "moka", "num-traits", + "openssl", "parse-display", "paste", "risingwave_common", "risingwave_expr_macro", - "risingwave_object_store", "risingwave_pb", "risingwave_udf", "smallvec", @@ -9105,6 +9335,7 @@ dependencies = [ "thiserror-ext", "tracing", "workspace-hack", + "zstd 0.13.0", ] [[package]] @@ -9113,10 +9344,11 @@ version = "1.7.0-alpha" dependencies = [ "aho-corasick", "anyhow", - "arrow-schema 49.0.0", + "arrow-schema 50.0.0", "async-trait", "auto_enums", "chrono", + "chrono-tz", "criterion", "expect-test", "fancy-regex", @@ -9126,9 +9358,11 @@ dependencies = [ "icelake", "itertools 0.12.0", "jsonbb", + "linkme", "madsim-tokio", "md5", "num-traits", + "openssl", "regex", "risingwave_common", "risingwave_expr", @@ -9162,33 +9396,36 @@ version = "1.7.0-alpha" dependencies = [ "anyhow", "arc-swap", - "arrow-schema 49.0.0", + "arrow-schema 50.0.0", "arrow-udf-wasm", "assert_matches", "async-recursion", "async-trait", "auto_enums", "auto_impl", - "base64 0.21.4", + "base64 0.22.0", "bk-tree", "bytes", "clap", - "ctor", "downcast-rs", "dyn-clone", "easy-ext", "educe 0.5.7", "either", "enum-as-inner", - "fixedbitset", + "fancy-regex", + "fixedbitset 0.5.0", "futures", "futures-async-stream", "iana-time-zone", + "icelake", "itertools 0.12.0", + "linkme", "madsim-tokio", "madsim-tonic", "maplit", "md5", + "memcomparable", "num-integer", "parking_lot 0.12.1", "parse-display", @@ -9203,19 +9440,22 @@ dependencies = [ "rand", "risingwave_batch", "risingwave_common", + "risingwave_common_heap_profiling", "risingwave_common_service", "risingwave_connector", + "risingwave_dml", "risingwave_expr", "risingwave_expr_impl", + "risingwave_frontend_macro", "risingwave_hummock_sdk", "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", - "risingwave_source", "risingwave_sqlparser", "risingwave_storage", "risingwave_udf", "risingwave_variables", + "rw_futures_util", "serde", "serde_json", "sha2", @@ -9227,6 +9467,16 @@ dependencies = [ "tracing", "uuid", "workspace-hack", + "zstd 0.13.0", +] + +[[package]] +name = "risingwave_frontend_macro" +version = "1.7.0-alpha" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.48", ] [[package]] @@ -9322,6 +9572,7 @@ dependencies = [ "cfg-or-panic", "chrono", "expect-test", + "fs-err", "futures", "itertools 0.12.0", "jni", @@ -9334,6 +9585,7 @@ dependencies = [ "risingwave_object_store", "risingwave_pb", "risingwave_storage", + "rw_futures_util", "serde", "serde_json", "thiserror", @@ -9366,7 +9618,6 @@ dependencies = [ "assert_matches", "async-trait", "aws-config", - "aws-sdk-ec2", "axum", "base64-url", "bytes", @@ -9411,6 +9662,8 @@ dependencies = [ "risingwave_rpc_client", "risingwave_sqlparser", "risingwave_test_runner", + "rw-aws-sdk-ec2", + "rw_futures_util", "scopeguard", "sea-orm", "serde", @@ -9442,6 +9695,8 @@ dependencies = [ name = "risingwave_meta_model_v2" version = "1.7.0-alpha" dependencies = [ + "prost 0.12.1", + "risingwave_common", "risingwave_hummock_sdk", "risingwave_pb", "sea-orm", @@ -9476,6 +9731,7 @@ dependencies = [ "sea-orm", "serde", "serde_json", + "thiserror-ext", "tracing", "workspace-hack", ] @@ -9501,6 +9757,7 @@ dependencies = [ "risingwave_pb", "sea-orm", "sync-point", + "thiserror-ext", "tokio-stream", "tracing", "workspace-hack", @@ -9526,6 +9783,7 @@ dependencies = [ "hyper-rustls", "hyper-tls", "itertools 0.12.0", + "madsim", "madsim-aws-sdk-s3", "madsim-tokio", "opendal", @@ -9566,7 +9824,7 @@ dependencies = [ "anyhow", "expect-test", "itertools 0.12.0", - "libtest-mimic", + "libtest-mimic 0.7.0", "madsim-tokio", "paste", "risingwave_expr_impl", @@ -9616,6 +9874,7 @@ dependencies = [ "risingwave_error", "risingwave_hummock_sdk", "risingwave_pb", + "rw_futures_util", "static_assertions", "thiserror", "thiserror-ext", @@ -9641,6 +9900,7 @@ dependencies = [ "opentelemetry", "opentelemetry-otlp", "opentelemetry-semantic-conventions", + "opentelemetry_sdk", "parking_lot 0.12.1", "pprof", "risingwave_common", @@ -9690,6 +9950,7 @@ dependencies = [ "risingwave_frontend", "risingwave_hummock_sdk", "risingwave_meta_node", + "risingwave_object_store", "risingwave_pb", "risingwave_rpc_client", "risingwave_sqlparser", @@ -9706,28 +9967,6 @@ dependencies = [ "tracing-subscriber", ] -[[package]] -name = "risingwave_source" -version = "1.7.0-alpha" -dependencies = [ - "anyhow", - "assert_matches", - "criterion", - "futures", - "futures-async-stream", - "itertools 0.12.0", - "madsim-tokio", - "parking_lot 0.12.1", - "paste", - "rand", - "risingwave_common", - "risingwave_connector", - "risingwave_pb", - "tempfile", - "tracing", - "workspace-hack", -] - [[package]] name = "risingwave_sqlparser" version = "1.7.0-alpha" @@ -9764,7 +10003,7 @@ dependencies = [ "clap", "expect-test", "itertools 0.12.0", - "libtest-mimic", + "libtest-mimic 0.7.0", "madsim-tokio", "rand", "rand_chacha", @@ -9776,6 +10015,7 @@ dependencies = [ "risingwave_pb", "risingwave_sqlparser", "similar", + "thiserror-ext", "tokio-postgres", "tracing", "tracing-subscriber", @@ -9835,7 +10075,7 @@ dependencies = [ "memcomparable", "moka", "more-asserts", - "nix 0.27.1", + "nix 0.28.0", "num-integer", "parking_lot 0.12.1", "procfs 0.16.0", @@ -9880,6 +10120,7 @@ dependencies = [ "auto_enums", "await-tree", "bytes", + "cfg-if", "criterion", "delta_btree_map", "educe 0.5.7", @@ -9898,7 +10139,7 @@ dependencies = [ "madsim-tonic", "maplit", "memcomparable", - "multimap 0.9.0", + "multimap 0.10.0", "parking_lot 0.12.1", "pin-project", "prometheus", @@ -9906,14 +10147,15 @@ dependencies = [ "rand", "risingwave_common", "risingwave_connector", + "risingwave_dml", "risingwave_expr", "risingwave_expr_impl", "risingwave_hummock_sdk", "risingwave_hummock_test", "risingwave_pb", "risingwave_rpc_client", - "risingwave_source", "risingwave_storage", + "rw_futures_util", "serde", "serde_json", "serde_yaml", @@ -9943,12 +10185,14 @@ dependencies = [ name = "risingwave_udf" version = "0.1.0" dependencies = [ - "arrow-array 49.0.0", + "arrow-array 50.0.0", "arrow-flight", - "arrow-schema 49.0.0", - "arrow-select 49.0.0", + "arrow-schema 50.0.0", + "arrow-select 50.0.0", "cfg-or-panic", + "futures", "futures-util", + "ginepro", "madsim-tokio", "madsim-tonic", "prometheus", @@ -10021,6 +10265,33 @@ dependencies = [ "retain_mut", ] +[[package]] +name = "rquickjs" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad7f63201fa6f2ff8173e4758ea552549d687d8f63003361a8b5c50f7c446ded" +dependencies = [ + "rquickjs-core", +] + +[[package]] +name = "rquickjs-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cad00eeddc0f88af54ee202c8385fb214fe0423897c056a7df8369fb482e3695" +dependencies = [ + "rquickjs-sys", +] + +[[package]] +name = "rquickjs-sys" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "120dbbc3296de9b96de8890091635d46f3506cd38b4e8f21800c386c035d64fa" +dependencies = [ + "cc", +] + [[package]] name = "rsa" version = "0.9.2" @@ -10090,9 +10361,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.33.0" +version = "1.34.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "076ba1058b036d3ca8bcafb1d54d0b0572e99d7ecd3e4222723e18ca8e9ca9a8" +checksum = "755392e1a2f77afd95580d3f0d0e94ac83eeeb7167552c9b5bca549e61a94d83" dependencies = [ "arrayvec", "borsh", @@ -10156,9 +10427,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.28" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72e572a5e8ca657d7366229cdde4bd14c4eb5499a9573d4d366fe1b599daa316" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ "bitflags 2.4.0", "errno", @@ -10199,7 +10470,7 @@ version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d3987094b1d07b653b7dfdc3f70ce9a1da9c51ac18c1b06b662e4f9a0e9f4b2" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", ] [[package]] @@ -10230,6 +10501,39 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "rw-aws-sdk-ec2" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80dba3602b267a7f9dcc546ccbf1d05752447773146253c7e344e2a320630b7f" +dependencies = [ + "aws-credential-types", + "aws-http", + "aws-runtime", + "aws-smithy-async", + "aws-smithy-http", + "aws-smithy-json", + "aws-smithy-query", + "aws-smithy-runtime", + "aws-smithy-runtime-api", + "aws-smithy-types", + "aws-smithy-xml", + "aws-types", + "fastrand 2.0.1", + "http 0.2.9", + "regex", + "tracing", +] + +[[package]] +name = "rw_futures_util" +version = "0.0.0" +dependencies = [ + "futures", + "pin-project-lite", + "tokio", +] + [[package]] name = "ryu" version = "1.0.15" @@ -10327,9 +10631,9 @@ dependencies = [ [[package]] name = "sea-orm" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61f6c7daef05dde3476d97001e11fca7a52b655aa3bf4fd610ab2da1176a2ed5" +checksum = "6632f499b80cc6aaa781b302e4c9fae663e0e3dcf2640e9d80034d5b10731efe" dependencies = [ "async-stream", "async-trait", @@ -10355,9 +10659,9 @@ dependencies = [ [[package]] name = "sea-orm-cli" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e3f0ff2fa5672e2e7314d107c6498a18e469beeb340a0ed84e3075fce73c2cd" +checksum = "465ea2308d4716837e9af4a2cff8e14c28135867a580bb93e9e03d408a3a6afb" dependencies = [ "chrono", "clap", @@ -10372,9 +10676,9 @@ dependencies = [ [[package]] name = "sea-orm-macros" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd90e73d5f5b184bad525767da29fbfec132b4e62ebd6f60d2f2737ec6468f62" +checksum = "ec13bfb4c4aef208f68dbea970dd40d13830c868aa8dcb4e106b956e6bb4f2fa" dependencies = [ "heck 0.4.1", "proc-macro2", @@ -10386,9 +10690,9 @@ dependencies = [ [[package]] name = "sea-orm-migration" -version = "0.12.2" +version = "0.12.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f673fcefb3a7e7b89a12b6c0e854ec0be14367635ac3435369c8ad7f11e09e" +checksum = "ac734b6e5610c2764056cc8495fbc293cd1c8ebe084fdfb74c3b0cdaaff9bb92" dependencies = [ "async-trait", "clap", @@ -10403,9 +10707,9 @@ dependencies = [ [[package]] name = "sea-query" -version = "0.30.1" +version = "0.30.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28c05a5bf6403834be253489bbe95fa9b1e5486bc843b61f60d26b5c9c1e244b" +checksum = "4166a1e072292d46dc91f31617c2a1cdaf55a8be4b5c9f4bf2ba248e3ac4999b" dependencies = [ "bigdecimal 0.3.1", "chrono", @@ -10450,9 +10754,9 @@ dependencies = [ [[package]] name = "sea-schema" -version = "0.14.1" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cd9561232bd1b82ea748b581f15909d11de0db6563ddcf28c5d908aee8282f1" +checksum = "30d148608012d25222442d1ebbfafd1228dbc5221baf4ec35596494e27a2394e" dependencies = [ "futures", "sea-query", @@ -10563,9 +10867,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4" [[package]] name = "serde" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63261df402c67811e9ac6def069e4786148c4563f4b50fd4bf30aa370d626b02" +checksum = "870026e60fa08c69f064aa766c10f10b1d62db9ccd4d0abb206472bee0ce3b32" dependencies = [ "serde_derive", ] @@ -10612,9 +10916,9 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.195" +version = "1.0.196" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "46fe8f8603d81ba86327b23a2e9cdf49e1255fb94a4c5f297f6ee0547178ea2c" +checksum = "33c85360c95e7d137454dc81d9a4ed2b8efd8fbe19cee57357b32b9771fccb67" dependencies = [ "proc-macro2", "quote", @@ -10634,9 +10938,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.113" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "69801b70b1c3dac963ecb03a364ba0ceda9cf60c71cfe475e99864759c8b8a79" dependencies = [ "itoa", "ryu", @@ -10705,11 +11009,11 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.4.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64cd236ccc1b7a29e7e2739f27c0b2dd199804abc4290e32f59f3b68d6405c23" +checksum = "1b0ed1662c5a68664f45b76d18deb0e234aff37207086803165c961eb695e981" dependencies = [ - "base64 0.21.4", + "base64 0.21.7", "chrono", "hex", "indexmap 1.9.3", @@ -10722,9 +11026,9 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.4.0" +version = "3.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93634eb5f75a2323b16de4748022ac4297f9e76b6dced2be287a099f41b5e788" +checksum = "568577ff0ef47b879f736cd66740e022f3672788cdf002a05a4e609ea5a6fb15" dependencies = [ "darling 0.20.3", "proc-macro2", @@ -10834,9 +11138,9 @@ dependencies = [ [[package]] name = "shlex" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7cee0529a6d40f580e7a5e6c495c8fbfe21b7b52795ed4bb5e62cdf92bc6380" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook" @@ -11009,9 +11313,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.11.1" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "942b4a808e05215192e39f4ab80813e599068285906cc91aa64f923db842bd5a" +checksum = "e6ecd384b10a64542d77071bd64bd7b231f4ed5940fba55e98c3de13824cf3d7" dependencies = [ "serde", ] @@ -11071,7 +11375,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "242f76c50fd18cbf098607090ade73a08d39cfd84ea835f3796a2c855223b19b" dependencies = [ "strum", - "strum_macros", + "strum_macros 0.25.3", ] [[package]] @@ -11152,7 +11456,7 @@ dependencies = [ "glob", "humantime", "itertools 0.11.0", - "libtest-mimic", + "libtest-mimic 0.6.1", "md-5", "owo-colors", "regex", @@ -11289,7 +11593,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ca69bf415b93b60b80dc8fda3cb4ef52b2336614d8da2de5456cc942a110482" dependencies = [ "atoi", - "base64 0.21.4", + "base64 0.21.7", "bigdecimal 0.3.1", "bitflags 2.4.0", "byteorder", @@ -11336,7 +11640,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a0db2df1b8731c3651e204629dd55e52adbae0462fa1bdcbed56a2302c18181e" dependencies = [ "atoi", - "base64 0.21.4", + "base64 0.21.7", "bigdecimal 0.3.1", "bitflags 2.4.0", "byteorder", @@ -11434,11 +11738,17 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "strsim" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" + [[package]] name = "structmeta" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ad9e09554f0456d67a69c1584c9798ba733a5b50349a6c0d0948710523922d" +checksum = "2e1575d8d40908d70f6fd05537266b90ae71b15dbbe7a8b7dffa2b759306d329" dependencies = [ "proc-macro2", "quote", @@ -11448,9 +11758,9 @@ dependencies = [ [[package]] name = "structmeta-derive" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a60bcaff7397072dca0017d1db428e30d5002e00b6847703e2e42005c95fbe00" +checksum = "152a0b65a590ff6c3da95cabe2353ee04e6167c896b28e3b14478c2636c922fc" dependencies = [ "proc-macro2", "quote", @@ -11463,7 +11773,7 @@ version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "290d54ea6f91c969195bdbcd7442c8c2a2ba87da8bf60a7ee86a235d4bc1e125" dependencies = [ - "strum_macros", + "strum_macros 0.25.3", ] [[package]] @@ -11479,6 +11789,19 @@ dependencies = [ "syn 2.0.48", ] +[[package]] +name = "strum_macros" +version = "0.26.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a3417fc93d76740d974a01654a09777cb500428cc874ca9f45edfe0c4d4cd18" +dependencies = [ + "heck 0.4.1", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.48", +] + [[package]] name = "subprocess" version = "0.2.9" @@ -11614,7 +11937,7 @@ dependencies = [ "cap-std", "fd-lock", "io-lifetimes 2.0.3", - "rustix 0.38.28", + "rustix 0.38.31", "windows-sys 0.48.0", "winx", ] @@ -11647,14 +11970,13 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.9.0" +version = "3.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" +checksum = "a365e8cd18e44762ef95d87f284f4b5cd04107fec2ff3052bd6a3e6069669e67" dependencies = [ "cfg-if", - "fastrand 2.0.0", - "redox_syscall 0.4.1", - "rustix 0.38.28", + "fastrand 2.0.1", + "rustix 0.38.31", "windows-sys 0.52.0", ] @@ -12069,7 +12391,7 @@ checksum = "3082666a3a6433f7f511c7192923fa1fe07c69332d3c6a2e6bb040b569199d5a" dependencies = [ "async-trait", "axum", - "base64 0.21.4", + "base64 0.21.7", "bytes", "futures-core", "futures-util", @@ -12098,7 +12420,7 @@ dependencies = [ "async-stream", "async-trait", "axum", - "base64 0.21.4", + "base64 0.21.7", "bytes", "flate2", "h2", @@ -12193,11 +12515,10 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.37" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ - "cfg-if", "log", "pin-project-lite", "tracing-attributes", @@ -12206,9 +12527,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.26" +version = "0.1.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f4f31f56159e98206da9efd823404b79b6ef3143b4a7ab76e67b1751b25a4ab" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" dependencies = [ "proc-macro2", "quote", @@ -12217,9 +12538,9 @@ dependencies = [ [[package]] name = "tracing-core" -version = "0.1.31" +version = "0.1.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54" dependencies = [ "once_cell", "valuable", @@ -12248,20 +12569,33 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-log" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3" +dependencies = [ + "log", + "once_cell", + "tracing-core", +] + [[package]] name = "tracing-opentelemetry" -version = "0.21.0" +version = "0.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75327c6b667828ddc28f5e3f169036cb793c3f588d83bf0f262a7f062ffed3c8" +checksum = "c67ac25c5407e7b961fafc6f7e9aa5958fd297aada2d20fa2ae1737357e55596" dependencies = [ + "js-sys", "once_cell", "opentelemetry", "opentelemetry_sdk", "smallvec", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.2.0", "tracing-subscriber", + "web-time", ] [[package]] @@ -12293,7 +12627,7 @@ dependencies = [ "time", "tracing", "tracing-core", - "tracing-log", + "tracing-log 0.1.3", "tracing-serde", ] @@ -12332,6 +12666,52 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622b09ce2fe2df4618636fb92176d205662f59803f39e70d1c333393082de96c" +[[package]] +name = "trust-dns-proto" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3119112651c157f4488931a01e586aa459736e9d6046d3bd9105ffb69352d374" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna 0.4.0", + "ipnet", + "once_cell", + "rand", + "smallvec", + "thiserror", + "tinyvec", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "trust-dns-resolver" +version = "0.23.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10a3e6c3aff1718b3c73e395d1f35202ba2ffa847c6a62eea0db8fb4cfe30be6" +dependencies = [ + "cfg-if", + "futures-util", + "ipconfig", + "lru-cache", + "once_cell", + "parking_lot 0.12.1", + "rand", + "resolv-conf", + "smallvec", + "thiserror", + "tokio", + "tracing", + "trust-dns-proto", +] + [[package]] name = "try-lock" version = "0.2.4" @@ -12458,6 +12838,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" +[[package]] +name = "unindent" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce" + [[package]] name = "unsafe-libyaml" version = "0.2.10" @@ -12483,7 +12869,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "31e6302e3bb753d46e83516cae55ae196fc0c309407cf11ab35cc51a4c2a4633" dependencies = [ "form_urlencoded", - "idna", + "idna 0.5.0", "percent-encoding", "serde", ] @@ -12502,9 +12888,9 @@ checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" [[package]] name = "uuid" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e395fcf16a7a3d8127ec99782007af141946b4795001f876d54fb0d55978560" +checksum = "f00cc9702ca12d3c81455259621e676d0f7251cec66a21e98fe2e9a37db93b2a" dependencies = [ "getrandom", "rand", @@ -12543,11 +12929,12 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "vergen" -version = "8.2.5" +version = "8.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85e7dc29b3c54a2ea67ef4f953d5ec0c4085035c0ae2d325be1c0d2144bd9f16" +checksum = "ec0d895592fa7710eba03fe072e614e3dc6a61ab76ae7ae10d2eb4a7ed5b00ca" dependencies = [ "anyhow", + "cfg-if", "rustversion", "time", ] @@ -12587,9 +12974,9 @@ checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" [[package]] name = "walkdir" -version = "2.4.0" +version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d71d857dc86794ca4c280d616f7da00d2dbfd8cd788846559a6813e6aa4b54ee" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" dependencies = [ "same-file", "winapi-util", @@ -12612,9 +12999,9 @@ checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "wasi-cap-std-sync" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154528979a211aa28d969846e883df75705809ed9bcc70aba61460683ea7355b" +checksum = "db014d2ced91f17d1f1a8f2b76d6ea8d731bc1dbc8c2bbaec689d6a242568e5d" dependencies = [ "anyhow", "async-trait", @@ -12626,18 +13013,18 @@ dependencies = [ "io-extras", "io-lifetimes 2.0.3", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "system-interface", "tracing", "wasi-common", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasi-common" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d888b611fee7d273dd057dc009d2dd3132736f36710ffd65657ac83628d1e3b" +checksum = "449d17849e3c83a931374442fe2deee4d6bd1ebf469719ef44192e9e82e19c89" dependencies = [ "anyhow", "bitflags 2.4.0", @@ -12645,14 +13032,20 @@ dependencies = [ "cap-std", "io-extras", "log", - "rustix 0.38.28", + "rustix 0.38.31", "thiserror", "tracing", "wasmtime", "wiggle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] +[[package]] +name = "wasite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8dad83b4f25e74f184f64c43b150b91efe7647395b42289f38e50566d82855b" + [[package]] name = "wasm-bindgen" version = "0.2.87" @@ -12763,9 +13156,9 @@ dependencies = [ [[package]] name = "wasmtime" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8e539fded2495422ea3c4dfa7beeddba45904eece182cf315294009e1a323bf" +checksum = "910fabce77e660f0e0e41cfd5f69fc8bf020a025f059718846e918db7177f469" dependencies = [ "anyhow", "async-trait", @@ -12797,43 +13190,43 @@ dependencies = [ "wasmtime-runtime", "wasmtime-winch", "wat", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-asm-macros" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "660ba9143e15a2acd921820df221b73aee256bd3ca2d208d73d8adc9587ccbb9" +checksum = "37288142e9b4a61655a3bcbdc7316c2e4bb9e776b10ce3dd758f8186b4469572" dependencies = [ "cfg-if", ] [[package]] name = "wasmtime-cache" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ce373743892002f9391c6741ef0cb0335b55ec899d874f311222b7e36f4594" +checksum = "45cbd74a636f09d2108f9405c79857f061e19323e4abeed22e837cfe7b08a22b" dependencies = [ "anyhow", - "base64 0.21.4", + "base64 0.21.7", "bincode 1.3.3", "directories-next", "log", - "rustix 0.38.28", + "rustix 0.38.31", "serde", "serde_derive", "sha2", "toml 0.5.11", - "windows-sys 0.48.0", + "windows-sys 0.52.0", "zstd 0.11.2+zstd.1.5.2", ] [[package]] name = "wasmtime-component-macro" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12ef32643324e564e1c359e9044daa06cbf90d7e2d6c99a738d17a12959f01a5" +checksum = "ad63de18eb42e586386b6091f787c82707cbd5ac5e9343216dba1976190cd03a" dependencies = [ "anyhow", "proc-macro2", @@ -12846,15 +13239,15 @@ dependencies = [ [[package]] name = "wasmtime-component-util" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c87d06c18d21a4818f354c00a85f4ebc62b2270961cd022968452b0e4dbed9d" +checksum = "7e0a160c0c44369aa4bee6d311a8e4366943bab1651040cc8b0fcec2c9eb8906" [[package]] name = "wasmtime-cranelift" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d648c8b4064a7911093b02237cd5569f71ca171d3a0a486bf80600b19e1cba2" +checksum = "3734cc01b7cd37bc62fdbcd9529ca9547440052d4b3886cfdec3b8081a5d3647" dependencies = [ "anyhow", "cfg-if", @@ -12877,9 +13270,9 @@ dependencies = [ [[package]] name = "wasmtime-cranelift-shared" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "290a89027688782da8ff60b12bb95695494b1874e0d0ba2ba387d23dace6d70c" +checksum = "e0eb33cd30c47844aa228d4d0030587e65c1108343f311fe9f7248b5bd9cb65c" dependencies = [ "anyhow", "cranelift-codegen", @@ -12893,9 +13286,9 @@ dependencies = [ [[package]] name = "wasmtime-environ" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61eb64fb3e0da883e2df4a13a81d6282e072336e6cb6295021d0f7ab2e352754" +checksum = "9a3a056b041fdea604f0972e2fae97958e7748d629a55180228348baefdfc217" dependencies = [ "anyhow", "cranelift-entity", @@ -12916,24 +13309,24 @@ dependencies = [ [[package]] name = "wasmtime-fiber" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40ecf1d3a838b0956b71ad3f8cb80069a228339775bf02dd35d86a5a68bbe443" +checksum = "43987d0977c07f15c3608c2f255870c127ffd19e35eeedb1ac1dccedf9932a42" dependencies = [ "anyhow", "cc", "cfg-if", - "rustix 0.38.28", + "rustix 0.38.31", "wasmtime-asm-macros", "wasmtime-versioned-export-macros", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f485336add49267d8859e8f8084d2d4b9a4b1564496b6f30ba5b168d50c10ceb" +checksum = "9b3e48395ac672b386ed588d97a9612aa13a345008f26466f0dfb2a91628aa9f" dependencies = [ "addr2line", "anyhow", @@ -12945,7 +13338,7 @@ dependencies = [ "log", "object", "rustc-demangle", - "rustix 0.38.28", + "rustix 0.38.31", "serde", "serde_derive", "target-lexicon", @@ -12953,37 +13346,37 @@ dependencies = [ "wasmtime-jit-debug", "wasmtime-jit-icache-coherence", "wasmtime-runtime", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-jit-debug" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65e119affec40edb2fab9044f188759a00c2df9c3017278d047012a2de1efb4f" +checksum = "dd21fd0f5ca68681d3d5b636eea00f182d0f9d764144469e9257fd7e3f55ae0e" dependencies = [ "object", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "wasmtime-versioned-export-macros", ] [[package]] name = "wasmtime-jit-icache-coherence" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b6d197fcc34ad32ed440e1f9552fd57d1f377d9699d31dee1b5b457322c1f8a" +checksum = "bdc26415bb89e9ccd3bdc498fef63aabf665c4c0dd710c107691deb9694955da" dependencies = [ "cfg-if", "libc", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-runtime" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "794b2bb19b99ef8322ff0dd9fe1ba7e19c41036dfb260b3f99ecce128c42ff92" +checksum = "0abddaf17912aabaf39be0802d5eba9a002e956e902d1ebd438a2fe1c88769a2" dependencies = [ "anyhow", "cc", @@ -12997,7 +13390,7 @@ dependencies = [ "memoffset", "paste", "psm", - "rustix 0.38.28", + "rustix 0.38.31", "sptr", "wasm-encoder", "wasmtime-asm-macros", @@ -13006,14 +13399,14 @@ dependencies = [ "wasmtime-jit-debug", "wasmtime-versioned-export-macros", "wasmtime-wmemcheck", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-types" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d995db8bb56f2cd8d2dc0ed5ffab94ffb435283b0fe6747f80f7aab40b2d06a1" +checksum = "b35a95cdc1433729085beab42c0a5c742b431f25b17c40d7718e46df63d5ffc7" dependencies = [ "cranelift-entity", "serde", @@ -13024,9 +13417,9 @@ dependencies = [ [[package]] name = "wasmtime-versioned-export-macros" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f55c5565959287c21dd0f4277ae3518dd2ae62679f655ee2dbc4396e19d210db" +checksum = "fad322733fe67e45743784d8b1df452bcb54f581572a4f1a646a4332deecbcc2" dependencies = [ "proc-macro2", "quote", @@ -13035,9 +13428,9 @@ dependencies = [ [[package]] name = "wasmtime-wasi" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccd8370078149d49a3a47e93741553fd79b700421464b6a27ca32718192ab130" +checksum = "902cc299b73655c36679b77efdfce4bb5971992f1a4a8a436dd3809a6848ff0e" dependencies = [ "anyhow", "async-trait", @@ -13055,7 +13448,7 @@ dependencies = [ "libc", "log", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", "system-interface", "thiserror", "tokio", @@ -13065,14 +13458,14 @@ dependencies = [ "wasi-common", "wasmtime", "wiggle", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] name = "wasmtime-winch" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c6f945ff9bad96e0a69973d74f193c19f627c8adbf250e7cb73ae7564b6cc8a" +checksum = "9e63aeca929f84560eec52c5af43bf5d623b92683b0195d9fb06da8ed860e092" dependencies = [ "anyhow", "cranelift-codegen", @@ -13087,9 +13480,9 @@ dependencies = [ [[package]] name = "wasmtime-wit-bindgen" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f328b2d4a690270324756e886ed5be3a4da4c00be0eea48253f4595ad068062b" +checksum = "41e5675998fdc74495afdd90ad2bd221206a258075b23048af0535a969b07893" dependencies = [ "anyhow", "heck 0.4.1", @@ -13099,9 +13492,9 @@ dependencies = [ [[package]] name = "wasmtime-wmemcheck" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67761d8f8c0b3c13a5d34356274b10a40baba67fe9cfabbfc379a8b414e45de2" +checksum = "b20a19e10d8cb50b45412fb21192982b7ce85c0122dc33bb71f1813e25dc6e52" [[package]] name = "wast" @@ -13143,6 +13536,16 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "web-time" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57099a701fb3a8043f993e8228dc24229c7b942e2b009a1b962e54489ba1d3bf" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "webpki-roots" version = "0.25.2" @@ -13158,24 +13561,31 @@ dependencies = [ "either", "home", "once_cell", - "rustix 0.38.28", + "rustix 0.38.31", ] [[package]] name = "whoami" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22fc3756b8a9133049b26c7f61ab35416c130e8c09b660f5b3958b446f52cc50" +checksum = "0fec781d48b41f8163426ed18e8fc2864c12937df9ce54c88ede7bd47270893e" dependencies = [ - "wasm-bindgen", + "redox_syscall 0.4.1", + "wasite", "web-sys", ] +[[package]] +name = "widestring" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "653f141f39ec16bba3c5abe400a0c60da7468261cc2cbf36805022876bc721a8" + [[package]] name = "wiggle" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0afb26cd3269289bb314a361ff0a6685e5ce793b62181a9fe3f81ace15051697" +checksum = "737728db69a7657a5f6a7bac445c02d8564d603d62c46c95edf928554e67d072" dependencies = [ "anyhow", "async-trait", @@ -13188,9 +13598,9 @@ dependencies = [ [[package]] name = "wiggle-generate" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cef2868fed7584d2b552fa317104858ded80021d23b073b2d682d3c932a027bd" +checksum = "2460c7163b79ffefd9a564eaeab0a5b0e84bb91afdfeeb84d36f304ddbe08982" dependencies = [ "anyhow", "heck 0.4.1", @@ -13203,9 +13613,9 @@ dependencies = [ [[package]] name = "wiggle-macro" -version = "16.0.0" +version = "17.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "31ae1ec11a17ea481539ee9a5719a278c9790d974060fbf71db4b2c05378780b" +checksum = "fa8d8412375ba8325d61fbae56dead51dabfaec85d620ce36427922fb9cece83" dependencies = [ "proc-macro2", "quote", @@ -13246,9 +13656,9 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "winch-codegen" -version = "0.14.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58e58c236a6abdd9ab454552b4f29e16cfa837a86897c1503313b2e62e7609ec" +checksum = "9d2b346bad5397b219b4ff0a8fa7230936061ff07c61f05d589d8d81e06fb7b2" dependencies = [ "anyhow", "cranelift-codegen", @@ -13569,6 +13979,7 @@ name = "workspace-hack" version = "1.7.0-alpha" dependencies = [ "ahash 0.8.6", + "aho-corasick", "allocator-api2", "anyhow", "async-std", @@ -13580,7 +13991,7 @@ dependencies = [ "aws-smithy-runtime", "aws-smithy-types", "axum", - "base64 0.21.4", + "base64 0.21.7", "bigdecimal 0.4.2", "bit-vec", "bitflags 2.4.0", @@ -13598,26 +14009,25 @@ dependencies = [ "digest", "either", "fail", - "fixedbitset", "flate2", "frunk_core", "futures", "futures-channel", "futures-core", + "futures-executor", "futures-io", "futures-sink", "futures-task", "futures-util", "generic-array", "governor", - "hashbrown 0.12.3", "hashbrown 0.13.2", "hashbrown 0.14.0", "hmac", "hyper", "indexmap 1.9.3", "indexmap 2.0.0", - "itertools 0.10.5", + "itertools 0.11.0", "jni", "lazy_static", "lexical-core", @@ -13632,14 +14042,16 @@ dependencies = [ "madsim-rdkafka", "madsim-tokio", "md-5", + "memchr", "mio", + "moka", "nom", "num-bigint", "num-integer", "num-iter", "num-traits", - "opentelemetry_api", - "opentelemetry_sdk", + "openssl", + "openssl-sys", "ordered-float 3.9.1", "parking_lot 0.12.1", "parking_lot_core 0.9.8", @@ -13657,13 +14069,13 @@ dependencies = [ "rand_core", "redis", "regex", - "regex-automata 0.4.1", - "regex-syntax 0.8.0", + "regex-automata 0.4.5", + "regex-syntax 0.8.2", "reqwest", "ring 0.16.20", "rust_decimal", "rustc-hash", - "rustix 0.38.28", + "rustix 0.38.31", "scopeguard", "sea-orm", "sea-query", @@ -13683,6 +14095,7 @@ dependencies = [ "subtle", "syn 1.0.109", "syn 2.0.48", + "target-lexicon", "time", "time-macros", "tinyvec", diff --git a/Cargo.toml b/Cargo.toml index 555e6a7d2a1e0..38ee801b39a9a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,12 +13,14 @@ members = [ "src/connector", "src/connector/with_options", "src/ctl", + "src/dml", "src/error", "src/expr/core", "src/expr/impl", "src/expr/macro", "src/expr/udf", "src/frontend", + "src/frontend/macro", "src/frontend/planner_test", "src/java_binding", "src/jni_core", @@ -33,7 +35,6 @@ members = [ "src/risedevtool", "src/risedevtool/config", "src/rpc_client", - "src/source", "src/sqlparser", "src/sqlparser/test_runner", "src/storage", @@ -51,6 +52,7 @@ members = [ "src/tests/sqlsmith", "src/tests/state_cleaning_test", "src/utils/delta_btree_map", + "src/utils/futures_util", "src/utils/local_stats_alloc", "src/utils/pgwire", "src/utils/runtime", @@ -88,7 +90,8 @@ aws-sdk-s3 = { version = "1", default-features = false, features = [ "rt-tokio", "rustls", ] } -aws-sdk-ec2 = { version = "1", default-features = false, features = [ +# To bump the version of aws-sdk-ec2, check the README of https://github.com/risingwavelabs/rw-aws-sdk-ec2 +aws-sdk-ec2 = { package = "rw-aws-sdk-ec2", version = "1", default-features = false, features = [ "rt-tokio", "rustls", ] } @@ -120,19 +123,21 @@ tonic = { package = "madsim-tonic", version = "0.4.1" } tonic-build = { package = "madsim-tonic-build", version = "0.4.2" } otlp-embedded = { git = "https://github.com/risingwavelabs/otlp-embedded", rev = "58c1f003484449d7c6dd693b348bf19dd44889cb" } prost = { version = "0.12" } -icelake = { git = "https://github.com/icelake-io/icelake", rev = "32c0bbf242f5c47b1e743f10577012fe7436c770", features = [ +icelake = { git = "https://github.com/icelake-io/icelake", rev = "54fd72fbd1dd8c592f05eeeb79223c8a6a33c297", features = [ "prometheus", ] } -arrow-array = "49" -arrow-arith = "49" -arrow-cast = "49" -arrow-schema = "49" -arrow-buffer = "49" -arrow-flight = "49" -arrow-select = "49" -arrow-ord = "49" -arrow-row = "49" -arrow-udf-wasm = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "f9a9e0d" } +arrow-array = "50" +arrow-arith = "50" +arrow-cast = "50" +arrow-schema = "50" +arrow-buffer = "50" +arrow-flight = "50" +arrow-select = "50" +arrow-ord = "50" +arrow-row = "50" +arrow-udf-js = "0.1" +arrow-udf-wasm = { version = "0.1.2", features = ["build"] } +arrow-udf-python = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "6c32f71" } arrow-array-deltalake = { package = "arrow-array", version = "48.0.1" } arrow-buffer-deltalake = { package = "arrow-buffer", version = "48.0.1" } arrow-cast-deltalake = { package = "arrow-cast", version = "48.0.1" } @@ -142,13 +147,20 @@ arrow-schema-deltalake = { package = "arrow-schema", version = "48.0.1" } deltalake = { git = "https://github.com/risingwavelabs/delta-rs", rev = "5c2dccd4640490202ffe98adbd13b09cef8e007b", features = [ "s3-no-concurrent-write", ] } -parquet = "49" +lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "95f347b" } +parquet = "50" thiserror-ext = "0.0.11" tikv-jemalloc-ctl = { git = "https://github.com/risingwavelabs/jemallocator.git", rev = "64a2d9" } tikv-jemallocator = { git = "https://github.com/risingwavelabs/jemallocator.git", features = [ "profiling", "stats", ], rev = "64a2d9" } +opentelemetry = "0.21" +opentelemetry-otlp = "0.14" +opentelemetry_sdk = { version = "0.21", default-features = false } +opentelemetry-semantic-conventions = "0.13" +tokio-util = "0.7" +tracing-opentelemetry = "0.22" risingwave_backup = { path = "./src/storage/backup" } risingwave_batch = { path = "./src/batch" } @@ -160,6 +172,7 @@ risingwave_compactor = { path = "./src/storage/compactor" } risingwave_compute = { path = "./src/compute" } risingwave_ctl = { path = "./src/ctl" } risingwave_connector = { path = "./src/connector" } +risingwave_dml = { path = "./src/dml" } risingwave_error = { path = "./src/error" } risingwave_expr = { path = "./src/expr/core" } risingwave_expr_impl = { path = "./src/expr/impl" } @@ -177,7 +190,6 @@ risingwave_object_store = { path = "./src/object_store" } risingwave_pb = { path = "./src/prost" } risingwave_rpc_client = { path = "./src/rpc_client" } risingwave_rt = { path = "./src/utils/runtime" } -risingwave_source = { path = "./src/source" } risingwave_sqlparser = { path = "./src/sqlparser" } risingwave_sqlsmith = { path = "./src/tests/sqlsmith" } risingwave_storage = { path = "./src/storage" } @@ -187,7 +199,7 @@ risingwave_udf = { path = "./src/expr/udf" } risingwave_variables = { path = "./src/utils/variables" } risingwave_java_binding = { path = "./src/java_binding" } risingwave_jni_core = { path = "src/jni_core" } -tokio-util = "0.7" +rw_futures_util = { path = "src/utils/futures_util" } [workspace.lints.rust] # `forbid` will also prevent the misuse of `#[allow(unused)]` @@ -237,6 +249,7 @@ lto = "thin" [profile.ci-release] inherits = "release" incremental = false +lto = "off" debug = "line-tables-only" split-debuginfo = "off" debug-assertions = true diff --git a/Makefile.toml b/Makefile.toml index c0135dc84c048..ed39ea0fc9425 100644 --- a/Makefile.toml +++ b/Makefile.toml @@ -29,12 +29,12 @@ env_scripts = [ set_env ENABLE_TELEMETRY "false" is_sanitizer_enabled = get_env ENABLE_SANITIZER -is_all_in_one_enabled = get_env ENABLE_ALL_IN_ONE is_hdfs_backend = get_env ENABLE_HDFS is_release = get_env ENABLE_RELEASE_PROFILE is_not_release = not ${is_release} is_dynamic_linking = get_env ENABLE_DYNAMIC_LINKING is_hummock_trace = get_env ENABLE_HUMMOCK_TRACE +is_python_udf_enabled = get_env ENABLE_PYTHON_UDF if ${is_sanitizer_enabled} set_env RISEDEV_CARGO_BUILD_EXTRA_ARGS "-Zbuild-std --target ${CARGO_MAKE_RUST_TARGET_TRIPLE}" @@ -45,13 +45,8 @@ else set_env RISEDEV_BUILD_TARGET_DIR "" end -if ${is_all_in_one_enabled} - set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd_all" - set_env RISEDEV_CTL_RUN_CMD "-- risectl" -else - set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd" - set_env RISEDEV_CTL_RUN_CMD "--bin risectl --" -end +set_env RISEDEV_CARGO_BUILD_CRATE "risingwave_cmd_all" +set_env RISEDEV_CTL_RUN_CMD "-- risectl" if ${is_hdfs_backend} set_env BUILD_HDFS_BACKEND_CMD "-p risingwave_object_store --features hdfs-backend" @@ -65,6 +60,11 @@ else set_env RISINGWAVE_FEATURE_FLAGS "--features rw-static-link" end +if ${is_python_udf_enabled} + flags = get_env RISINGWAVE_FEATURE_FLAGS + set_env RISINGWAVE_FEATURE_FLAGS "${flags} --features embedded-python-udf" +end + if ${is_hummock_trace} set_env BUILD_HUMMOCK_TRACE_CMD "-p risingwave_storage --features hm-trace" else @@ -262,31 +262,10 @@ set -e python -mwebbrowser file://$(pwd)/target/doc/index.html ''' -[tasks.link-standalone-binaries] -private = true -category = "RiseDev - Build" -description = "Link standalone cmds to RiseDev bin" -condition = { env_not_set = ["ENABLE_ALL_IN_ONE"] } -script = ''' -#!/usr/bin/env bash -set -e -rm -f "${PREFIX_BIN}/compute-node" -rm -f "${PREFIX_BIN}/meta-node" -rm -f "${PREFIX_BIN}/frontend" -rm -f "${PREFIX_BIN}/compactor" -rm -f "${PREFIX_BIN}/risectl" -ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/compute-node" "${PREFIX_BIN}/compute-node" -ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/meta-node" "${PREFIX_BIN}/meta-node" -ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/frontend" "${PREFIX_BIN}/frontend" -ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/compactor" "${PREFIX_BIN}/compactor" -ln -s "$(pwd)/target/${RISEDEV_BUILD_TARGET_DIR}${BUILD_MODE_DIR}/risectl" "${PREFIX_BIN}/risectl" -''' - [tasks.link-all-in-one-binaries] private = true category = "RiseDev - Build" description = "Link all-in-one cmds to RiseDev bin" -condition = { env_set = ["ENABLE_ALL_IN_ONE"] } script = ''' #!/usr/bin/env bash set -e @@ -333,13 +312,7 @@ script = ''' #!/usr/bin/env bash set -e -binaries=() - -if [[ "$ENABLE_ALL_IN_ONE" == "true" ]]; then - binaries=("risingwave") -else - binaries=("meta-node" "compute-node" "frontend" "compactor") -fi +binaries=("risingwave") set -ex echo -n "${binaries[*]}" | parallel -d ' ' \ @@ -350,7 +323,6 @@ echo -n "${binaries[*]}" | parallel -d ' ' \ private = true category = "RiseDev - Build" description = "Link all binaries to .bin" -condition = { env_set = ["ENABLE_ALL_IN_ONE"] } script = ''' #!/usr/bin/env bash set -e @@ -369,7 +341,6 @@ category = "RiseDev - Build" description = "Copy RisngWave binaries to bin" condition = { env_set = ["ENABLE_BUILD_RUST"] } dependencies = [ - "link-standalone-binaries", "link-all-in-one-binaries", "link-user-bin", "codesign-binaries", @@ -621,6 +592,25 @@ if [ ! -f "${RC_ENV_FILE}" ]; then fi ''' +[tasks.psql-env] +category = "RiseDev - Start/Stop" +description = "Dump env configuration for psql" +dependencies = ["check-risedev-env-file"] +env_files = ["${PREFIX_CONFIG}/risedev-env"] +script = ''' +#!/usr/bin/env bash +cat < "${PREFIX_CONFIG}/psql-env" +export PGHOST=$RW_FRONTEND_LISTEN_ADDRESS +export PGPORT=$RW_FRONTEND_PORT +export PGUSER=root +export PGDATABASE=dev +EOF + +echo "psql environment file is created at ${PREFIX_CONFIG}/psql-env" +echo "You can source this file to use psql with default connection parameters." +echo " $(tput setaf 4)source ${PREFIX_CONFIG}/psql-env$(tput sgr0)" +''' + [tasks.psql] category = "RiseDev - Start/Stop" description = "Run local psql client with default connection parameters. You can pass extra arguments to psql." @@ -757,10 +747,10 @@ tmux list-windows -t risedev -F "#{window_name} #{pane_id}" \ if [[ -n $(tmux list-windows -t risedev | grep kafka) ]]; then echo "kill kafka" - kill_kafka + kill_kafka || true echo "kill zookeeper" - kill_zookeeper + kill_zookeeper || true # Kill their tmux sessions tmux list-windows -t risedev -F "#{pane_id}" | xargs -I {} tmux send-keys -t {} C-c C-d @@ -919,10 +909,10 @@ cargo build \ -p risingwave_common \ -p risingwave_compute \ -p risingwave_connector \ + -p risingwave_dml \ -p risingwave_frontend \ -p risingwave_meta \ -p risingwave_object_store \ - -p risingwave_source \ -p risingwave_storage \ -p risingwave_stream \ -p pgwire \ @@ -944,10 +934,10 @@ cargo nextest run \ -p risingwave_common \ -p risingwave_compute \ -p risingwave_connector \ + -p risingwave_dml \ -p risingwave_frontend \ -p risingwave_meta \ -p risingwave_object_store \ - -p risingwave_source \ -p risingwave_storage \ -p risingwave_stream \ -p pgwire \ @@ -1000,10 +990,10 @@ cargo check \ -p risingwave_common \ -p risingwave_compute \ -p risingwave_connector \ + -p risingwave_dml \ -p risingwave_frontend \ -p risingwave_meta \ -p risingwave_object_store \ - -p risingwave_source \ -p risingwave_storage \ -p risingwave_stream \ -p pgwire \ @@ -1305,6 +1295,11 @@ command = "target/${BUILD_MODE_DIR}/risedev-dev" args = ["${@}"] description = "Clean data and start a full RisingWave dev cluster using risedev-dev" +[tasks.ci-kill-no-dump-logs] +category = "RiseDev - CI" +dependencies = ["k", "check-logs", "wait-processes-exit"] +description = "Kill cluster and check logs, do not dump logs" + [tasks.ci-kill] category = "RiseDev - CI" dependencies = ["k", "l", "check-logs", "wait-processes-exit"] diff --git a/README.md b/README.md index 11793ce064647..c1a6f2c12f1b1 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@
-### 🌊Stream Processing Redefined. +### 🌊 Reimagine Stream Processing.
@@ -40,12 +40,6 @@ > Slack - - Docker - -RisingWave is a distributed SQL streaming database engineered to provide the simplest and most cost-efficient approach for processing and managing streaming data with utmost reliability. +RisingWave is a Postgres-compatible streaming database engineered to provide the simplest and most cost-efficient approach for processing, analyzing, and managing real-time event streaming data. ![RisingWave](https://github.com/risingwavelabs/risingwave-docs/blob/main/docs/images/new_archi_grey.png) -## Try it out in 5 minutes -**Docker** -``` -docker run -it --pull=always -p 4566:4566 -p 5691:5691 risingwavelabs/risingwave:latest playground -``` -**Mac** -``` -brew tap risingwavelabs/risingwave -brew install risingwave -risingwave playground -``` -**Ubuntu** -``` -wget https://github.com/risingwavelabs/risingwave/releases/download/v1.5.0/risingwave-v1.5.0-x86_64-unknown-linux-all-in-one.tar.gz -tar xvf risingwave-v1.5.0-x86_64-unknown-linux-all-in-one.tar.gz -./risingwave playground -``` -Now connect to RisingWave using `psql`: -``` -psql -h localhost -p 4566 -d dev -U root +## Try it out in 60 seconds + +Install RisingWave: +```shell +curl https://risingwave.com/sh | sh ``` -Learn more at [Quick Start](https://docs.risingwave.com/docs/current/get-started/). + +Then follow the prompts to start and connect to RisingWave. + +To learn about other installation options such as Docker, see [Quick Start](https://docs.risingwave.com/docs/current/get-started/). ## Production deployments -For **single-node Docker deployments**, please refer to [Docker Compose](https://docs.risingwave.com/docs/current/risingwave-trial/?method=docker-compose). +For **single-node deployment**, please refer to [Docker Compose](https://docs.risingwave.com/docs/current/risingwave-docker-compose/). -For **Kubernetes deployments**, please refer to [Kubernetes with Helm](https://docs.risingwave.com/docs/current/risingwave-k8s-helm/) or [Kubernetes with Operator](https://docs.risingwave.com/docs/current/risingwave-kubernetes/). +For **distributed deployment**, please refer to [Kubernetes with Helm](https://docs.risingwave.com/docs/current/risingwave-k8s-helm/) or [Kubernetes with Operator](https://docs.risingwave.com/docs/current/risingwave-kubernetes/). **RisingWave Cloud** the easiest way to run a fully-fledged RisingWave cluster. Try it out for free at: [cloud.risingwave.com](https://cloud.risingwave.com). @@ -135,17 +117,15 @@ RisingWave is fundamentally a database that **extends beyond basic streaming dat * Schema change * Processing of semi-structured data +## In-production use cases +Within your data stack, RisingWave can assist with: -## RisingWave's limitations -RisingWave isn’t a panacea for all data engineering hurdles. It has its own set of limitations: -* **No programmable interfaces** - * RisingWave does not provide low-level APIs in languages like Java and Scala, and does not allow users to manage internal states manually (unless you want to hack!). _For coding in Java, Python, and other languages, please consider using RisingWave's [User-Defined Functions (UDF)](https://docs.risingwave.com/docs/current/user-defined-functions/)_. -* **No support for transaction processing** - * RisingWave isn’t cut out for transactional workloads, thus it’s not a viable substitute for operational databases dedicated to transaction processing. _However, it supports [read-only transactions](https://docs.risingwave.com/docs/current/transactions/#read-only-transactions), ensuring data freshness and consistency. It also comprehends the transactional semantics of upstream database [Change Data Capture (CDC)](https://docs.risingwave.com/docs/current/transactions/#transactions-within-a-cdc-table)_. - +* Processing and transforming event streaming data in real time +* Offloading event-driven queries (e.g., materialized views, triggers) from operational databases +* Performing real-time ETL (Extract, Transform, Load) +* Supporting real-time feature stores -## In-production use cases -Like other stream processing systems, the primary use cases of RisingWave include monitoring, alerting, real-time dashboard reporting, streaming ETL (Extract, Transform, Load), machine learning feature engineering, and more. It has already been adopted in fields such as financial trading, manufacturing, new media, logistics, gaming, and more. Check out [customer stories](https://www.risingwave.com/use-cases/). +RisingWave is extensively utilized in real-time applications such as monitoring, alerting, dashboard reporting, machine learning, among others. It has already been adopted in fields such as financial trading, manufacturing, new media, logistics, gaming, and more. Check out [customer stories](https://www.risingwave.com/use-cases/). ## Community diff --git a/backwards-compat-tests/scripts/run_local.sh b/backwards-compat-tests/scripts/run_local.sh index 81545f70b77a8..76b1a4f333458 100755 --- a/backwards-compat-tests/scripts/run_local.sh +++ b/backwards-compat-tests/scripts/run_local.sh @@ -37,9 +37,6 @@ ENABLE_KAFKA=true # Fetch risingwave binary from release. ENABLE_BUILD_RUST=true -# Ensure it will link the all-in-one binary from our release. -ENABLE_ALL_IN_ONE=true - # ENABLE_RELEASE_PROFILE=true EOF } diff --git a/backwards-compat-tests/scripts/utils.sh b/backwards-compat-tests/scripts/utils.sh index 1afbf08dd4441..3bbd9b06ca42c 100644 --- a/backwards-compat-tests/scripts/utils.sh +++ b/backwards-compat-tests/scripts/utils.sh @@ -49,6 +49,23 @@ kill_zookeeper() { wait_zookeeper_exit } +wait_for_process() { + process_name="$1" + + while pgrep -x "$process_name" > /dev/null; do + echo "Process $process_name is still running... Wait for 1 sec" + sleep 1 + done +} + +wait_all_process_exit() { + wait_for_process meta-node + wait_for_process compute-node + wait_for_process frontend + wait_for_process compactor + echo "All processes has exited." +} + # Older versions of RW may not gracefully kill kafka. # So we duplicate the definition here. kill_cluster() { @@ -75,6 +92,7 @@ kill_cluster() { tmux kill-session -t risedev test $? -eq 0 || { echo "Failed to stop all RiseDev components."; exit 1; } + wait_all_process_exit } run_sql () { @@ -103,19 +121,21 @@ insert_json_kafka() { local JSON=$1 echo "$JSON" | "$KAFKA_PATH"/bin/kafka-console-producer.sh \ --topic backwards_compat_test_kafka_source \ - --bootstrap-server localhost:29092 + --bootstrap-server localhost:29092 \ + --property "parse.key=true" \ + --property "key.separator=," } seed_json_kafka() { - insert_json_kafka '{"timestamp": "2023-07-28 07:11:00", "user_id": 1, "page_id": 1, "action": "gtrgretrg"}' - insert_json_kafka '{"timestamp": "2023-07-28 07:11:00", "user_id": 2, "page_id": 1, "action": "fsdfgerrg"}' - insert_json_kafka '{"timestamp": "2023-07-28 07:11:00", "user_id": 3, "page_id": 1, "action": "sdfergtth"}' - insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 4, "page_id": 2, "action": "erwerhghj"}' - insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 5, "page_id": 2, "action": "kiku7ikkk"}' - insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 6, "page_id": 3, "action": "6786745ge"}' - insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 7, "page_id": 3, "action": "fgbgfnyyy"}' - insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 8, "page_id": 4, "action": "werwerwwe"}' - insert_json_kafka '{"timestamp": "2023-07-28 06:54:00", "user_id": 9, "page_id": 4, "action": "yjtyjtyyy"}' + insert_json_kafka '{"user_id": 1},{"timestamp": "2023-07-28 07:11:00", "user_id": 1, "page_id": 1, "action": "gtrgretrg"}' + insert_json_kafka '{"user_id": 2},{"timestamp": "2023-07-28 07:11:00", "user_id": 2, "page_id": 1, "action": "fsdfgerrg"}' + insert_json_kafka '{"user_id": 3},{"timestamp": "2023-07-28 07:11:00", "user_id": 3, "page_id": 1, "action": "sdfergtth"}' + insert_json_kafka '{"user_id": 4},{"timestamp": "2023-07-28 06:54:00", "user_id": 4, "page_id": 2, "action": "erwerhghj"}' + insert_json_kafka '{"user_id": 5},{"timestamp": "2023-07-28 06:54:00", "user_id": 5, "page_id": 2, "action": "kiku7ikkk"}' + insert_json_kafka '{"user_id": 6},{"timestamp": "2023-07-28 06:54:00", "user_id": 6, "page_id": 3, "action": "6786745ge"}' + insert_json_kafka '{"user_id": 7},{"timestamp": "2023-07-28 06:54:00", "user_id": 7, "page_id": 3, "action": "fgbgfnyyy"}' + insert_json_kafka '{"user_id": 8},{"timestamp": "2023-07-28 06:54:00", "user_id": 8, "page_id": 4, "action": "werwerwwe"}' + insert_json_kafka '{"user_id": 9},{"timestamp": "2023-07-28 06:54:00", "user_id": 9, "page_id": 4, "action": "yjtyjtyyy"}' } # https://stackoverflow.com/a/4024263 @@ -225,6 +245,12 @@ seed_old_cluster() { create_kafka_topic seed_json_kafka sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/seed.slt" + # use the old syntax for version at most 1.5.4 + if version_le "$OLD_VERSION" "1.5.4" ; then + sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/upsert/deprecate_upsert.slt" + else + sqllogictest -d dev -h localhost -p 4566 "$TEST_DIR/kafka/upsert/include_key_as.slt" + fi echo "--- KAFKA TEST: wait 5s for kafka to process data" sleep 5 diff --git a/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt b/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt new file mode 100644 index 0000000000000..55cfce886455d --- /dev/null +++ b/backwards-compat-tests/slt/kafka/upsert/deprecate_upsert.slt @@ -0,0 +1,16 @@ +statement ok +CREATE TABLE IF NOT EXISTS kafka_table +( + action varchar, + user_id integer, + obj_id integer, + name varchar, + page_id integer, + age integer +) +WITH ( + connector='kafka', + topic='backwards_compat_test_kafka_source', + properties.bootstrap.server='localhost:29092', + scan.startup.mode='earliest', +) FORMAT UPSERT ENCODE JSON; \ No newline at end of file diff --git a/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt b/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt new file mode 100644 index 0000000000000..36ef426574223 --- /dev/null +++ b/backwards-compat-tests/slt/kafka/upsert/include_key_as.slt @@ -0,0 +1,18 @@ +statement ok +CREATE TABLE IF NOT EXISTS kafka_table +( + action varchar, + user_id integer, + obj_id integer, + name varchar, + page_id integer, + age integer, + primary key (_rw_key) +) +INCLUDE key as _rw_key +WITH ( + connector='kafka', + topic='backwards_compat_test_kafka_source', + properties.bootstrap.server='localhost:29092', + scan.startup.mode='earliest', +) FORMAT UPSERT ENCODE JSON; \ No newline at end of file diff --git a/backwards-compat-tests/slt/kafka/validate_restart.slt b/backwards-compat-tests/slt/kafka/validate_restart.slt index 7058b118f4d20..6d853007b9829 100644 --- a/backwards-compat-tests/slt/kafka/validate_restart.slt +++ b/backwards-compat-tests/slt/kafka/validate_restart.slt @@ -50,3 +50,16 @@ werwerwwe 8 NULL NULL 4 NULL yjtyjtyyy 9 NULL NULL 4 NULL yjtyjtyyy 9 NULL NULL 4 NULL +# kafka_table should do the upsert and overwrite the existing records +query I rowsort +SELECT action, user_id, obj_id, name, page_id, age, _rw_key FROM kafka_table; +---- +6786745ge 6 NULL NULL 3 NULL \x7b22757365725f6964223a20367d +erwerhghj 4 NULL NULL 2 NULL \x7b22757365725f6964223a20347d +fgbgfnyyy 7 NULL NULL 3 NULL \x7b22757365725f6964223a20377d +fsdfgerrg 2 NULL NULL 1 NULL \x7b22757365725f6964223a20327d +gtrgretrg 1 NULL NULL 1 NULL \x7b22757365725f6964223a20317d +kiku7ikkk 5 NULL NULL 2 NULL \x7b22757365725f6964223a20357d +sdfergtth 3 NULL NULL 1 NULL \x7b22757365725f6964223a20337d +werwerwwe 8 NULL NULL 4 NULL \x7b22757365725f6964223a20387d +yjtyjtyyy 9 NULL NULL 4 NULL \x7b22757365725f6964223a20397d diff --git a/ci/Dockerfile b/ci/Dockerfile index 810a3289b66a8..d1b193a002356 100644 --- a/ci/Dockerfile +++ b/ci/Dockerfile @@ -11,10 +11,16 @@ ENV LANG en_US.utf8 # Use AWS ubuntu mirror RUN sed -i 's|http://archive.ubuntu.com/ubuntu|http://us-east-2.ec2.archive.ubuntu.com/ubuntu/|g' /etc/apt/sources.list RUN apt-get update -yy && \ - DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip \ + DEBIAN_FRONTEND=noninteractive apt-get -y install make build-essential cmake protobuf-compiler curl parallel python3 python3-pip software-properties-common \ openssl libssl-dev libsasl2-dev libcurl4-openssl-dev pkg-config bash openjdk-11-jdk wget unzip git tmux lld postgresql-client kafkacat netcat mysql-client \ maven zstd libzstd-dev locales -yy \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ +# Install Python 3.12 +RUN add-apt-repository ppa:deadsnakes/ppa -y && \ + apt-get update -yy && \ + DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy && \ + rm -rf /var/lib/{apt,dpkg,cache,log}/ +ENV PYO3_PYTHON=python3.12 SHELL ["/bin/bash", "-c"] @@ -29,6 +35,7 @@ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --no-mo ENV PATH /root/.cargo/bin/:$PATH RUN rustup show +RUN rustup default `rustup show active-toolchain | awk '{print $1}'` RUN curl -sSL "https://github.com/bufbuild/buf/releases/download/v1.4.0/buf-$(uname -s)-$(uname -m).tar.gz" | \ tar -xvzf - -C /usr/local --strip-components 1 @@ -41,17 +48,19 @@ RUN curl -sSL https://install.python-poetry.org | python3 - # add required rustup components RUN rustup component add rustfmt llvm-tools-preview clippy +RUN rustup target add wasm32-wasi ENV CARGO_REGISTRIES_CRATES_IO_PROTOCOL=sparse # install build tools RUN curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash RUN cargo binstall -y --no-symlinks cargo-llvm-cov cargo-nextest cargo-hakari cargo-sort cargo-cache cargo-audit \ - cargo-make@0.36.10 \ + cargo-make@0.37.9 \ sqllogictest-bin@0.19.1 \ sccache@0.7.4 \ && cargo cache -a \ && rm -rf "/root/.cargo/registry/index" \ && rm -rf "/root/.cargo/registry/cache" \ && rm -rf "/root/.cargo/git/db" +RUN cargo install cargo-dylint@2.6.0 dylint-link@2.6.0 RUN cargo uninstall cargo-binstall cargo-cache diff --git a/ci/build-ci-image.sh b/ci/build-ci-image.sh index bbdc5d7bfa9ed..6ad98102c6ce1 100755 --- a/ci/build-ci-image.sh +++ b/ci/build-ci-image.sh @@ -10,7 +10,7 @@ cat ../rust-toolchain # shellcheck disable=SC2155 # REMEMBER TO ALSO UPDATE ci/docker-compose.yml -export BUILD_ENV_VERSION=v20240104_1 +export BUILD_ENV_VERSION=v20240223 export BUILD_TAG="public.ecr.aws/x5u3w5h6/rw-build-env:${BUILD_ENV_VERSION}" diff --git a/ci/docker-compose.yml b/ci/docker-compose.yml index 0ebb9e77eeb20..829aaf2dd0a65 100644 --- a/ci/docker-compose.yml +++ b/ci/docker-compose.yml @@ -71,7 +71,7 @@ services: retries: 5 source-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223 depends_on: - mysql - db @@ -81,24 +81,29 @@ services: - ..:/risingwave sink-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223 depends_on: - mysql - db - message_queue - elasticsearch - clickhouse-server - - pulsar + - redis-server + - pulsar-server + - cassandra-server + - starrocks-fe-server + - starrocks-be-server volumes: - ..:/risingwave + rw-build-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223 volumes: - ..:/risingwave ci-flamegraph-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223 # NOTE(kwannoel): This is used in order to permit # syscalls for `nperf` (perf_event_open), # so it can do CPU profiling. @@ -109,14 +114,14 @@ services: - ..:/risingwave regress-test-env: - image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240104_1 + image: public.ecr.aws/x5u3w5h6/rw-build-env:v20240223 depends_on: db: condition: service_healthy volumes: - ..:/risingwave - release-env: + release-env-x86: # build binaries on a earlier Linux distribution (therefore with earlier version GLIBC) # See https://github.com/risingwavelabs/risingwave/issues/4556 for more details. # @@ -132,6 +137,12 @@ services: volumes: - ..:/mnt + release-env-arm: + image: quay.io/pypa/manylinux2014_aarch64 + working_dir: /mnt + volumes: + - ..:/mnt + elasticsearch: container_name: elasticsearch image: docker.elastic.co/elasticsearch/elasticsearch:7.11.0 @@ -153,10 +164,63 @@ services: expose: - 9009 -# Temporary workaround for json schema registry test since redpanda only supports -# protobuf/avro schema registry. Should be removed after the support. -# Related tracking issue: -# https://github.com/redpanda-data/redpanda/issues/1878 + redis-server: + container_name: redis-server + image: 'redis:latest' + expose: + - 6379 + ports: + - 6378:6379 + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 5s + timeout: 30s + retries: 50 + + cassandra-server: + container_name: cassandra-server + image: cassandra:4.0 + ports: + - 9042:9042 + environment: + - CASSANDRA_CLUSTER_NAME=cloudinfra + + starrocks-fe-server: + container_name: starrocks-fe-server + image: starrocks/fe-ubuntu:3.1.7 + hostname: starrocks-fe-server + command: + /opt/starrocks/fe/bin/start_fe.sh + ports: + - 28030:8030 + - 29020:9020 + - 29030:9030 + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9030"] + interval: 5s + timeout: 5s + retries: 30 + + starrocks-be-server: + image: starrocks/be-ubuntu:3.1.7 + command: + - /bin/bash + - -c + - | + sleep 15s; mysql --connect-timeout 2 -h starrocks-fe-server -P9030 -uroot -e "alter system add backend \"starrocks-be-server:9050\";" + /opt/starrocks/be/bin/start_be.sh + ports: + - 28040:8040 + - 29050:9050 + hostname: starrocks-be-server + container_name: starrocks-be-server + depends_on: + - starrocks-fe-server + +# # Temporary workaround for json schema registry test since redpanda only supports +# # protobuf/avro schema registry. Should be removed after the support. +# # Related tracking issue: +# # https://github.com/redpanda-data/redpanda/issues/1878 zookeeper: container_name: zookeeper image: confluentinc/cp-zookeeper:latest @@ -195,8 +259,8 @@ services: KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9093,PLAINTEXT_INTERNAL://localhost:29093 KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - pulsar: - container_name: pulsar + pulsar-server: + container_name: pulsar-server image: apachepulsar/pulsar:latest command: bin/pulsar standalone ports: diff --git a/ci/plugins/upload-failure-logs-zipped/hooks/post-command b/ci/plugins/upload-failure-logs-zipped/hooks/post-command new file mode 100755 index 0000000000000..28e8eab77827c --- /dev/null +++ b/ci/plugins/upload-failure-logs-zipped/hooks/post-command @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +set -euo pipefail + +if [ $BUILDKITE_COMMAND_EXIT_STATUS -ne 0 ]; then + mv .risingwave/log risedev-logs + zip -q -r risedev-logs.zip risedev-logs/ + buildkite-agent artifact upload risedev-logs.zip + REGRESS_TEST_DIR="$PWD/src/tests/regress/output/results/" + if [ -d "$REGRESS_TEST_DIR" ]; then + mkdir regress-test-logs && cp src/tests/regress/output/results/* regress-test-logs/ + zip -q -r regress-test.zip regress-test-logs/ + buildkite-agent artifact upload regress-test-logs.zip + fi + if [ -e "$PWD/connector-node.log" ]; then + buildkite-agent artifact upload "$PWD/connector-node.log" + fi +fi \ No newline at end of file diff --git a/ci/risedev-components.ci.benchmark.env b/ci/risedev-components.ci.benchmark.env index 59136a30477ea..67171fe10bc28 100644 --- a/ci/risedev-components.ci.benchmark.env +++ b/ci/risedev-components.ci.benchmark.env @@ -1,7 +1,6 @@ RISEDEV_CONFIGURED=true ENABLE_MINIO=true ENABLE_ETCD=true -ENABLE_ALL_IN_ONE=true ENABLE_KAFKA=true ENABLE_BUILD_RUST=true ENABLE_RELEASE_PROFILE=true diff --git a/ci/risedev-components.ci.env b/ci/risedev-components.ci.env index 48adc7eb2594a..7157083eb871a 100644 --- a/ci/risedev-components.ci.env +++ b/ci/risedev-components.ci.env @@ -1,4 +1,3 @@ RISEDEV_CONFIGURED=true ENABLE_MINIO=true ENABLE_ETCD=true -ENABLE_ALL_IN_ONE=true diff --git a/ci/risedev-components.ci.source.env b/ci/risedev-components.ci.source.env index ccdd18b7ab9c4..255b873eab94d 100644 --- a/ci/risedev-components.ci.source.env +++ b/ci/risedev-components.ci.source.env @@ -1,6 +1,5 @@ RISEDEV_CONFIGURED=true ENABLE_MINIO=true ENABLE_ETCD=true -ENABLE_ALL_IN_ONE=true ENABLE_KAFKA=true ENABLE_PUBSUB=true diff --git a/ci/rust-toolchain b/ci/rust-toolchain index e8c05a601edc0..b1f2df70a8d0f 100644 --- a/ci/rust-toolchain +++ b/ci/rust-toolchain @@ -1,7 +1,7 @@ # To update toolchain, do the following: # 1. update this file -# 2. update lints/rust-toolchain, lints/Cargo.toml -# 3. update ci/build-ci-image.sh and ci/docker-compose.yml to build a new CI image +# 2. update ci/build-ci-image.sh and ci/docker-compose.yml to build a new CI image +# 3. (optional) **follow the instructions in lints/README.md** to update the toolchain and dependencies for lints [toolchain] channel = "nightly-2023-12-26" diff --git a/ci/scripts/backfill-test.sh b/ci/scripts/backfill-test.sh index 3050b94be714b..4769d7c5d229d 100755 --- a/ci/scripts/backfill-test.sh +++ b/ci/scripts/backfill-test.sh @@ -32,4 +32,4 @@ download_and_prepare_rw "$profile" source ################ TESTS -./ci/scripts/run-backfill-tests.sh +BUILDKITE=${BUILDKITE:-} profile=$profile ./ci/scripts/run-backfill-tests.sh diff --git a/ci/scripts/backwards-compat-test.sh b/ci/scripts/backwards-compat-test.sh index 0c013574d4185..84fda2e2f2b0f 100755 --- a/ci/scripts/backwards-compat-test.sh +++ b/ci/scripts/backwards-compat-test.sh @@ -37,6 +37,8 @@ source backwards-compat-tests/scripts/utils.sh ################################### Main configure_rw() { +VERSION="$1" + echo "--- Setting up cluster config" cat < risedev-profiles.user.yml full-without-monitoring: @@ -61,12 +63,13 @@ ENABLE_KAFKA=true # Fetch risingwave binary from release. ENABLE_BUILD_RUST=false -# Ensure it will link the all-in-one binary from our release. -ENABLE_ALL_IN_ONE=true - # Use target/debug for simplicity. ENABLE_RELEASE_PROFILE=false EOF + +if version_le "${VERSION:-}" "1.7.0" ; then + echo "ENABLE_ALL_IN_ONE=true" >> risedev-components.user.env +fi } configure_rw_build() { @@ -94,9 +97,6 @@ ENABLE_KAFKA=true # Make sure that it builds ENABLE_BUILD_RUST=true -# Ensure it will link the all-in-one binary from our release. -ENABLE_ALL_IN_ONE=true - # Use target/debug for simplicity. ENABLE_RELEASE_PROFILE=false EOF @@ -122,7 +122,7 @@ setup_old_cluster() { echo "--- Start cluster on tag $OLD_VERSION" git config --global --add safe.directory /risingwave - configure_rw + configure_rw $OLD_VERSION fi } @@ -144,7 +144,10 @@ main() { seed_old_cluster "$OLD_VERSION" setup_new_cluster - configure_rw + # Assume we use the latest version, so we just set to some large number. + # The current $NEW_VERSION as of this change is 1.7.0, so we can't use that. + # See: https://github.com/risingwavelabs/risingwave/pull/15448 + configure_rw "99.99.99" validate_new_cluster "$NEW_VERSION" } diff --git a/ci/scripts/build-other.sh b/ci/scripts/build-other.sh index e303a5f5160cb..2311e5164fe74 100755 --- a/ci/scripts/build-other.sh +++ b/ci/scripts/build-other.sh @@ -8,7 +8,6 @@ source ci/scripts/common.sh echo "--- Build Rust UDF" cd e2e_test/udf/wasm -rustup target add wasm32-wasi cargo build --release cd ../../.. diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 5fed2cfeedf1b..91b26bee8d383 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -53,6 +53,7 @@ cargo build \ -p risingwave_compaction_test \ -p risingwave_e2e_extended_mode_test \ $RISINGWAVE_FEATURE_FLAGS \ + --features embedded-python-udf \ --profile "$profile" diff --git a/ci/scripts/check-dylint.sh b/ci/scripts/check-dylint.sh new file mode 100755 index 0000000000000..77dc5dbff71f0 --- /dev/null +++ b/ci/scripts/check-dylint.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +source ci/scripts/common.sh +unset RUSTC_WRAPPER # disable sccache, see https://github.com/mozilla/sccache/issues/861 + +echo "--- Run dylint check (dev, all features)" +# Instead of `-D warnings`, we only deny warnings from our own lints. This is because... +# - Warnings from `check` or `clippy` are already checked in `check.sh`. +# - The toolchain used for linting could be slightly different from the one used to +# compile RisingWave. Warnings from `rustc` itself may produce false positives. +DYLINT_RUSTFLAGS="-A warnings -D rw_warnings" cargo dylint --all -- --all-targets --all-features --locked diff --git a/ci/scripts/check.sh b/ci/scripts/check.sh index b854da84c0b70..f52a1b4fd3e8a 100755 --- a/ci/scripts/check.sh +++ b/ci/scripts/check.sh @@ -30,6 +30,10 @@ sccache --zero-stats echo "--- Run clippy check (release)" cargo clippy --release --all-targets --features "rw-static-link" --locked -- -D warnings +echo "--- Run cargo check on building the release binary (release)" +cargo check -p risingwave_cmd_all --features "rw-static-link" --profile release +cargo check -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release + echo "--- Show sccache stats" sccache --show-stats sccache --zero-stats diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh old mode 100644 new mode 100755 index 5c710b8607f5b..35601c72e4e62 --- a/ci/scripts/common.sh +++ b/ci/scripts/common.sh @@ -87,3 +87,16 @@ function download_and_prepare_rw() { cargo make pre-start-dev cargo make --allow-private link-all-in-one-binaries } + +function filter_stack_trace() { + # Only keep first 3 lines of backtrace: 0-2. + echo "filtering stack trace for $1" + touch tmp + cat "$1" \ + | sed -E '/ [1-9][0-9]+:/d' \ + | sed -E '/ [3-9]+:/d' \ + | sed -E '/ at .rustc/d' \ + | sed -E '/ at ...cargo/d' > tmp + cp tmp "$1" + rm tmp +} \ No newline at end of file diff --git a/ci/scripts/cron-e2e-test.sh b/ci/scripts/cron-e2e-test.sh index 52c7a6faa366c..434fc6dea2985 100755 --- a/ci/scripts/cron-e2e-test.sh +++ b/ci/scripts/cron-e2e-test.sh @@ -6,5 +6,4 @@ set -euo pipefail source ci/scripts/common.sh export RUN_COMPACTION=0; export RUN_META_BACKUP=1; -export RUN_DELETE_RANGE=1; source ci/scripts/run-e2e-test.sh diff --git a/ci/scripts/deterministic-e2e-test.sh b/ci/scripts/deterministic-e2e-test.sh index 1681c35711c80..68a41fafe7433 100755 --- a/ci/scripts/deterministic-e2e-test.sh +++ b/ci/scripts/deterministic-e2e-test.sh @@ -49,7 +49,7 @@ echo "--- deterministic simulation e2e, ci-3cn-2fe, parallel, batch" seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -j 16 ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/parallel-batch-{}.log && rm $LOGDIR/parallel-batch-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, fuzzing (pre-generated-queries)" -timeout 7m seq 64 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log' +timeout 10m seq 64 | parallel MADSIM_TEST_SEED={} './risingwave_simulation --run-sqlsmith-queries ./src/tests/sqlsmith/tests/sqlsmith-query-snapshots/{} 2> $LOGDIR/fuzzing-{}.log && rm $LOGDIR/fuzzing-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe, e2e extended mode test" seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation -e 2> $LOGDIR/extended-{}.log && rm $LOGDIR/extended-{}.log' diff --git a/ci/scripts/deterministic-recovery-test.sh b/ci/scripts/deterministic-recovery-test.sh index 30c41603c9e60..b14cbce36cbbe 100755 --- a/ci/scripts/deterministic-recovery-test.sh +++ b/ci/scripts/deterministic-recovery-test.sh @@ -14,21 +14,70 @@ risingwave_meta::manager::catalog=debug,\ risingwave_meta::rpc::ddl_controller=debug,\ risingwave_meta::barrier::mod=debug,\ risingwave_simulation=debug" + +# Extra logs you can enable if the existing trace does not give enough info. +#risingwave_stream::executor::backfill=trace, +#risingwave_meta::barrier::progress=debug, + +# ========= Some tips for debugging recovery tests ========= +# 1. If materialized view failed to create after multiple retries +# - Check logs to see where the materialized view creation was stuck. +# 1. Is it stuck at waiting for backfill executor response? +# In that case perhaps some backfill logic is flawed, add more trace in backfill to debug. +# 2. Is it stuck at waiting for backfill executor to finish? +# In that case perhaps some part of the backfill loop is slow / stuck. +# 3. Is it stuck at waiting for some executor to report progress? +# In that case perhaps the tracking of backfill's progress in meta is flawed. + export LOGDIR=.risingwave/log mkdir -p $LOGDIR +filter_stack_trace_for_all_logs() { + # Defined in `common.sh` + for log in "${LOGDIR}"/*.log; do + filter_stack_trace $log + done +} + +trap filter_stack_trace_for_all_logs ERR + echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, background_ddl" -seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} ./e2e_test/background_ddl/sim/basic.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' +seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ +--kill \ +--kill-rate=${KILL_RATE} \ +${USE_ARRANGEMENT_BACKFILL:-} \ +./e2e_test/background_ddl/sim/basic.slt \ +2> $LOGDIR/recovery-background-ddl-{}.log && rm $LOGDIR/recovery-background-ddl-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, ddl" -seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --background-ddl-rate=${BACKGROUND_DDL_RATE} ./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' +seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ +--kill \ +--kill-rate=${KILL_RATE} \ +--background-ddl-rate=${BACKGROUND_DDL_RATE} \ +${USE_ARRANGEMENT_BACKFILL:-} \ +./e2e_test/ddl/\*\*/\*.slt 2> $LOGDIR/recovery-ddl-{}.log && rm $LOGDIR/recovery-ddl-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, streaming" -seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --background-ddl-rate=${BACKGROUND_DDL_RATE} ./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log' +seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ +--kill \ +--kill-rate=${KILL_RATE} \ +--background-ddl-rate=${BACKGROUND_DDL_RATE} \ +${USE_ARRANGEMENT_BACKFILL:-} \ +./e2e_test/streaming/\*\*/\*.slt 2> $LOGDIR/recovery-streaming-{}.log && rm $LOGDIR/recovery-streaming-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, batch" -seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --background-ddl-rate=${BACKGROUND_DDL_RATE} ./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log' +seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ +--kill \ +--kill-rate=${KILL_RATE} \ +--background-ddl-rate=${BACKGROUND_DDL_RATE} \ +${USE_ARRANGEMENT_BACKFILL:-} \ +./e2e_test/batch/\*\*/\*.slt 2> $LOGDIR/recovery-batch-{}.log && rm $LOGDIR/recovery-batch-{}.log' echo "--- deterministic simulation e2e, ci-3cn-2fe-3meta, recovery, kafka source,sink" -seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation --kill --kill-rate=${KILL_RATE} --kafka-datadir=./scripts/source/test_data ./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log' +seq $TEST_NUM | parallel MADSIM_TEST_SEED={} './risingwave_simulation \ +--kill \ +--kill-rate=${KILL_RATE} \ +--kafka-datadir=./scripts/source/test_data \ +${USE_ARRANGEMENT_BACKFILL:-} \ +./e2e_test/source/basic/kafka\*.slt 2> $LOGDIR/recovery-source-{}.log && rm $LOGDIR/recovery-source-{}.log' \ No newline at end of file diff --git a/ci/scripts/docker-hdfs.sh b/ci/scripts/docker-hdfs.sh old mode 100644 new mode 100755 diff --git a/ci/scripts/e2e-cassandra-sink-test.sh b/ci/scripts/e2e-cassandra-sink-test.sh new file mode 100755 index 0000000000000..c393d510d19a2 --- /dev/null +++ b/ci/scripts/e2e-cassandra-sink-test.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +source ci/scripts/common.sh + +# prepare environment +export CONNECTOR_LIBS_PATH="./connector-node/libs" + +while getopts 'p:' opt; do + case ${opt} in + p ) + profile=$OPTARG + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + ;; + esac +done +shift $((OPTIND -1)) + +download_and_prepare_rw "$profile" source + +echo "--- Download connector node package" +buildkite-agent artifact download risingwave-connector.tar.gz ./ +mkdir ./connector-node +tar xf ./risingwave-connector.tar.gz -C ./connector-node + +echo "--- starting risingwave cluster" +cargo make ci-start ci-sink-test +sleep 1 + +echo "--- create cassandra table" +curl https://downloads.apache.org/cassandra/4.1.3/apache-cassandra-4.1.3-bin.tar.gz --output apache-cassandra-4.1.3-bin.tar.gz +tar xfvz apache-cassandra-4.1.3-bin.tar.gz +cd apache-cassandra-4.1.3/bin +export CQLSH_HOST=cassandra-server +export CQLSH_PORT=9042 +./cqlsh -e "CREATE KEYSPACE demo WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1};use demo; +CREATE table demo_bhv_table(v1 int primary key,v2 smallint,v3 bigint,v4 float,v5 double,v6 text,v7 date,v8 timestamp,v9 boolean);" + +echo "--- testing sinks" +cd ../../ +sqllogictest -p 4566 -d dev './e2e_test/sink/cassandra_sink.slt' +sleep 1 +cd apache-cassandra-4.1.3/bin +./cqlsh -e "COPY demo.demo_bhv_table TO './query_result.csv' WITH HEADER = false AND ENCODING = 'UTF-8';" + +if cat ./query_result.csv | awk -F "," '{ + exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01.000+0000" && $9 == "False\r"); }'; then + echo "Cassandra sink check passed" +else + cat ./query_result.csv + echo "The output is not as expected." + exit 1 +fi + +echo "--- Kill cluster" +cd ../../ +cargo make ci-kill \ No newline at end of file diff --git a/ci/scripts/e2e-clickhouse-sink-test.sh b/ci/scripts/e2e-clickhouse-sink-test.sh index 51720b0e1d09b..5443d4e53b7fd 100755 --- a/ci/scripts/e2e-clickhouse-sink-test.sh +++ b/ci/scripts/e2e-clickhouse-sink-test.sh @@ -24,34 +24,35 @@ shift $((OPTIND -1)) download_and_prepare_rw "$profile" source echo "--- starting risingwave cluster" -cargo make ci-start ci-clickhouse-test +cargo make ci-start ci-sink-test sleep 1 echo "--- create clickhouse table" curl https://clickhouse.com/ | sh sleep 2 -./clickhouse client --host=clickhouse-server --port=9000 --query="CREATE table demo_test(v1 Int32,v2 Int64,v3 String)ENGINE = ReplacingMergeTree PRIMARY KEY (v1);" +./clickhouse client --host=clickhouse-server --port=9000 --query="CREATE table demo_test(v1 Int32,v2 Int64,v3 String,v4 Enum16('A'=1,'B'=2))ENGINE = ReplacingMergeTree PRIMARY KEY (v1);" echo "--- testing sinks" sqllogictest -p 4566 -d dev './e2e_test/sink/clickhouse_sink.slt' -sleep 1 +sleep 5 ./clickhouse client --host=clickhouse-server --port=9000 --query="select * from demo_test FORMAT CSV;" > ./query_result.csv # check sink destination using shell if cat ./query_result.csv | sort | awk -F "," '{ -if ($1 == 1 && $2 == 50 && $3 == "\"1-50\"") c1++; - if ($1 == 13 && $2 == 2 && $3 == "\"13-2\"") c2++; - if ($1 == 2 && $2 == 2 && $3 == "\"2-2\"") c3++; - if ($1 == 21 && $2 == 2 && $3 == "\"21-2\"") c4++; - if ($1 == 3 && $2 == 2 && $3 == "\"3-2\"") c5++; - if ($1 == 5 && $2 == 2 && $3 == "\"5-2\"") c6++; - if ($1 == 8 && $2 == 2 && $3 == "\"8-2\"") c7++; } +if ($1 == 1 && $2 == 50 && $3 == "\"1-50\"" && $4 == "\"A\"") c1++; + if ($1 == 13 && $2 == 2 && $3 == "\"13-2\"" && $4 == "\"B\"") c2++; + if ($1 == 2 && $2 == 2 && $3 == "\"2-2\"" && $4 == "\"B\"") c3++; + if ($1 == 21 && $2 == 2 && $3 == "\"21-2\"" && $4 == "\"A\"") c4++; + if ($1 == 3 && $2 == 2 && $3 == "\"3-2\"" && $4 == "\"A\"") c5++; + if ($1 == 5 && $2 == 2 && $3 == "\"5-2\"" && $4 == "\"B\"") c6++; + if ($1 == 8 && $2 == 2 && $3 == "\"8-2\"" && $4 == "\"A\"") c7++; } END { exit !(c1 == 1 && c2 == 1 && c3 == 1 && c4 == 1 && c5 == 1 && c6 == 1 && c7 == 1); }'; then echo "Clickhouse sink check passed" else echo "The output is not as expected." + cat ./query_result.csv exit 1 fi diff --git a/ci/scripts/e2e-deltalake-sink-rust-test.sh b/ci/scripts/e2e-deltalake-sink-rust-test.sh index 71ff1eede8e4d..cc0c287e8b572 100755 --- a/ci/scripts/e2e-deltalake-sink-rust-test.sh +++ b/ci/scripts/e2e-deltalake-sink-rust-test.sh @@ -32,8 +32,7 @@ mkdir ./connector-node tar xf ./risingwave-connector.tar.gz -C ./connector-node echo "--- starting risingwave cluster" -mkdir -p .risingwave/log -cargo make ci-start ci-deltalake-test +cargo make ci-start ci-sink-test sleep 1 # prepare minio deltalake sink diff --git a/ci/scripts/e2e-doris-sink-test.sh b/ci/scripts/e2e-doris-sink-test.sh new file mode 100755 index 0000000000000..30bfdaf129e26 --- /dev/null +++ b/ci/scripts/e2e-doris-sink-test.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +source ci/scripts/common.sh + +while getopts 'p:' opt; do + case ${opt} in + p ) + profile=$OPTARG + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + ;; + esac +done +shift $((OPTIND -1)) + +download_and_prepare_rw "$profile" source + +echo "--- starting risingwave cluster" +cargo make ci-start ci-sink-test +sleep 1 + +echo "--- create doris table" +apt-get update -y && apt-get install -y mysql-client +sleep 2 +mysql -uroot -P 9030 -h doris-fe-server -e "CREATE database demo;use demo; +CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 datev2,v8 datetime,v9 boolean) UNIQUE KEY(\`v1\`) +DISTRIBUTED BY HASH(\`v1\`) BUCKETS 1 +PROPERTIES ( + \"replication_allocation\" = \"tag.location.default: 1\" +); +CREATE USER 'users'@'%' IDENTIFIED BY '123456'; +GRANT ALL ON *.* TO 'users'@'%';" +sleep 2 + +echo "--- testing sinks" +sqllogictest -p 4566 -d dev './e2e_test/sink/doris_sink.slt' +sleep 1 +mysql -uroot -P 9030 -h doris-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv + + +if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ + exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0); }'; then + echo "Doris sink check passed" +else + cat ./query_result.csv + echo "The output is not as expected." + exit 1 +fi + +echo "--- Kill cluster" +cargo make ci-kill \ No newline at end of file diff --git a/ci/scripts/e2e-elasticsearch-sink-test.sh b/ci/scripts/e2e-elasticsearch-sink-test.sh old mode 100644 new mode 100755 diff --git a/ci/scripts/e2e-iceberg-sink-v2-test.sh b/ci/scripts/e2e-iceberg-sink-v2-test.sh index 0e8054a4946af..c3bac96c9654e 100755 --- a/ci/scripts/e2e-iceberg-sink-v2-test.sh +++ b/ci/scripts/e2e-iceberg-sink-v2-test.sh @@ -42,6 +42,8 @@ bash ./start_spark_connect_server.sh "$HOME"/.local/bin/poetry run python main.py -t ./test_case/no_partition_upsert.toml "$HOME"/.local/bin/poetry run python main.py -t ./test_case/partition_append_only.toml "$HOME"/.local/bin/poetry run python main.py -t ./test_case/partition_upsert.toml +"$HOME"/.local/bin/poetry run python main.py -t ./test_case/range_partition_append_only.toml +"$HOME"/.local/bin/poetry run python main.py -t ./test_case/range_partition_upsert.toml echo "--- Kill cluster" diff --git a/ci/scripts/e2e-pulsar-sink-test.sh b/ci/scripts/e2e-pulsar-sink-test.sh index ee8848832f940..f942ad945b3e9 100755 --- a/ci/scripts/e2e-pulsar-sink-test.sh +++ b/ci/scripts/e2e-pulsar-sink-test.sh @@ -21,7 +21,7 @@ shift $((OPTIND -1)) download_and_prepare_rw "$profile" source echo "--- starting risingwave cluster" -cargo make ci-start ci-pulsar-test +cargo make ci-start ci-sink-test sleep 1 echo "--- waiting until pulsar is healthy" diff --git a/ci/scripts/e2e-redis-sink-test.sh b/ci/scripts/e2e-redis-sink-test.sh new file mode 100755 index 0000000000000..cf64662db4051 --- /dev/null +++ b/ci/scripts/e2e-redis-sink-test.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +source ci/scripts/common.sh + +while getopts 'p:' opt; do + case ${opt} in + p ) + profile=$OPTARG + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + ;; + esac +done +shift $((OPTIND -1)) + +download_and_prepare_rw "$profile" source + +echo "--- starting risingwave cluster" +cargo make ci-start ci-sink-test +apt-get update -y && apt-get install -y redis-server +sleep 1 + +echo "--- testing sinks" +sqllogictest -p 4566 -d dev './e2e_test/sink/redis_sink.slt' +sleep 1 + +redis-cli -h redis-server -p 6379 get {\"v1\":1} >> ./query_result.txt +redis-cli -h redis-server -p 6379 get V1:1 >> ./query_result.txt + +# check sink destination using shell +if cat ./query_result.txt | tr '\n' '\0' | xargs -0 -n1 bash -c '[[ "$0" == "{\"v1\":1,\"v2\":1,\"v3\":1,\"v4\":1.100000023841858,\"v5\":1.2,\"v6\":\"test\",\"v7\":734869,\"v8\":\"2013-01-01T01:01:01.000000Z\",\"v9\":false}" || "$0" == "V2:1,V3:1" ]]'; then + echo "Redis sink check passed" +else + cat ./query_result.txt + echo "The output is not as expected." + exit 1 +fi + +echo "--- Kill cluster" +cargo make ci-kill \ No newline at end of file diff --git a/ci/scripts/e2e-sink-test.sh b/ci/scripts/e2e-sink-test.sh index c20f422532b79..300012680a198 100755 --- a/ci/scripts/e2e-sink-test.sh +++ b/ci/scripts/e2e-sink-test.sh @@ -109,7 +109,6 @@ else fi echo "--- testing elasticsearch sink" -chmod +x ./ci/scripts/e2e-elasticsearch-sink-test.sh ./ci/scripts/e2e-elasticsearch-sink-test.sh if [ $? -eq 0 ]; then echo "elasticsearch sink check passed" diff --git a/ci/scripts/e2e-source-test.sh b/ci/scripts/e2e-source-test.sh index 64144d051ad58..ec04a1d6863cf 100755 --- a/ci/scripts/e2e-source-test.sh +++ b/ci/scripts/e2e-source-test.sh @@ -5,6 +5,15 @@ set -euo pipefail source ci/scripts/common.sh +# Arguments: +# $1: subject name +# $2: schema file path +function register_schema_registry() { + curl -X POST http://message_queue:8081/subjects/$1/versions \ + -H ‘Content-Type: application/vnd.schemaregistry.v1+json’ \ + --data-binary @<(jq -n --arg schema “$(cat $2)†‘{schemaType: “PROTOBUFâ€, schema: $schema}’) +} + # prepare environment export CONNECTOR_LIBS_PATH="./connector-node/libs" @@ -115,12 +124,13 @@ export RISINGWAVE_CI=true RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ cargo make ci-start ci-1cn-1fe python3 -m pip install requests protobuf confluent-kafka -python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 20 +python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 20 user echo "make sure google/protobuf/source_context.proto is NOT in schema registry" curl --silent 'http://message_queue:8081/subjects'; echo # curl --silent --head -X GET 'http://message_queue:8081/subjects/google%2Fprotobuf%2Fsource_context.proto/versions' | grep 404 curl --silent 'http://message_queue:8081/subjects' | grep -v 'google/protobuf/source_context.proto' sqllogictest -p 4566 -d dev './e2e_test/schema_registry/pb.slt' +sqllogictest -p 4566 -d dev './e2e_test/schema_registry/alter_sr.slt' echo "--- Kill cluster" cargo make ci-kill diff --git a/ci/scripts/e2e-starrocks-sink-test.sh b/ci/scripts/e2e-starrocks-sink-test.sh new file mode 100755 index 0000000000000..256f4448f9198 --- /dev/null +++ b/ci/scripts/e2e-starrocks-sink-test.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euo pipefail + +source ci/scripts/common.sh + +while getopts 'p:' opt; do + case ${opt} in + p ) + profile=$OPTARG + ;; + \? ) + echo "Invalid Option: -$OPTARG" 1>&2 + exit 1 + ;; + : ) + echo "Invalid option: $OPTARG requires an argument" 1>&2 + ;; + esac +done +shift $((OPTIND -1)) + +download_and_prepare_rw "$profile" source + +echo "--- starting risingwave cluster" +cargo make ci-start ci-sink-test +sleep 1 + + +echo "--- create starrocks table" +apt-get update -y && apt-get install -y mysql-client +sleep 2 +mysql -uroot -P 9030 -h starrocks-fe-server -e "CREATE database demo;use demo; +CREATE table demo_bhv_table(v1 int,v2 smallint,v3 bigint,v4 float,v5 double,v6 string,v7 date,v8 datetime,v9 boolean,v10 json) ENGINE=OLAP +PRIMARY KEY(\`v1\`) +DISTRIBUTED BY HASH(\`v1\`) properties(\"replication_num\" = \"1\"); +CREATE USER 'users'@'%' IDENTIFIED BY '123456'; +GRANT ALL ON *.* TO 'users'@'%';" +sleep 2 + +echo "--- testing sinks" +sqllogictest -p 4566 -d dev './e2e_test/sink/starrocks_sink.slt' +sleep 1 +mysql -uroot -P 9030 -h starrocks-fe-server -e "select * from demo.demo_bhv_table" > ./query_result.csv + + +if cat ./query_result.csv | sed '1d; s/\t/,/g' | awk -F "," '{ + exit !($1 == 1 && $2 == 1 && $3 == 1 && $4 == 1.1 && $5 == 1.2 && $6 == "test" && $7 == "2013-01-01" && $8 == "2013-01-01 01:01:01" && $9 == 0 && $10 = "{"v101": 100}"); }'; then + echo "Starrocks sink check passed" +else + cat ./query_result.csv + echo "The output is not as expected." + exit 1 +fi + +echo "--- Kill cluster" +cargo make ci-kill \ No newline at end of file diff --git a/ci/scripts/gen-flamegraph.sh b/ci/scripts/gen-flamegraph.sh index 11abf2290d6f1..62e1543bd0e5b 100755 --- a/ci/scripts/gen-flamegraph.sh +++ b/ci/scripts/gen-flamegraph.sh @@ -97,7 +97,7 @@ install_all() { promql --version echo ">>> Installing Kafka" - wget https://downloads.apache.org/kafka/3.4.1/kafka_2.13-3.4.1.tgz + wget https://archive.apache.org/dist/kafka/3.4.1/kafka_2.13-3.4.1.tgz tar -zxvf kafka_2.13-3.4.1.tgz echo ">>> Installing nexmark bench" @@ -159,7 +159,6 @@ ENABLE_KAFKA=true ENABLE_COMPUTE_TRACING=true ENABLE_BUILD_RUST=true ENABLE_RELEASE_PROFILE=true -ENABLE_ALL_IN_ONE=true EOF popd } @@ -425,4 +424,4 @@ main() { # popd } -main "$@" \ No newline at end of file +main "$@" diff --git a/ci/scripts/gen-integration-test-yaml.py b/ci/scripts/gen-integration-test-yaml.py index 8451290a93c93..8f39ab6edb180 100644 --- a/ci/scripts/gen-integration-test-yaml.py +++ b/ci/scripts/gen-integration-test-yaml.py @@ -15,6 +15,7 @@ 'mysql-sink': ['json'], 'postgres-sink': ['json'], 'iceberg-cdc': ['json'], + 'iceberg-sink': ['none'], 'twitter': ['json', 'protobuf'], 'twitter-pulsar': ['json'], 'debezium-mysql': ['json'], @@ -51,6 +52,8 @@ def gen_pipeline_steps(): command: ci/scripts/integration-tests.sh -c {test_case} -f {test_format} timeout_in_minutes: 30 retry: *auto-retry + concurrency: 10 + concurrency_group: 'integration-test/run' plugins: - seek-oss/aws-sm#v2.3.1: env: diff --git a/ci/scripts/integration-tests.sh b/ci/scripts/integration-tests.sh index 9eed69256e50e..ef6c024ac9db0 100755 --- a/ci/scripts/integration-tests.sh +++ b/ci/scripts/integration-tests.sh @@ -56,6 +56,11 @@ fi echo "--- install postgresql" sudo yum install -y postgresql15 +echo "--- install poetry" +curl -sSL https://install.python-poetry.org | POETRY_VERSION=1.8.0 python3 - + + + echo "--- download rwctest-key" aws secretsmanager get-secret-value --secret-id "gcp-buildkite-rwctest-key" --region us-east-2 --query "SecretString" --output text >gcp-rwctest.json @@ -79,13 +84,6 @@ docker ps echo "--- check if the ingestion is successful" # extract the type of upstream source,e.g. mysql,postgres,etc upstream=$(echo ${case} | cut -d'-' -f 1) -if [ "${upstream}" == "mysql" ]; then - echo "install mysql" - sudo rpm -Uvh https://dev.mysql.com/get/mysql80-community-release-el9-1.noarch.rpm - sudo dnf -y install mysql-community-server -fi - -export PGPASSWORD=123456 python3 check_data.py ${case} ${upstream} echo "--- clean Demos" diff --git a/ci/scripts/notify.py b/ci/scripts/notify.py index 72450243af641..8d59214698d6d 100755 --- a/ci/scripts/notify.py +++ b/ci/scripts/notify.py @@ -82,11 +82,11 @@ "redis-sink-json": ["xinhao"], "big-query-sink-json": ["xinhao"], "vector-json": ["tao"], - "doris-sink": ["xinhao"], - "starrocks-sink": ["xinhao"], - "deltalake-sink": ["xinhao"], - "pinot-sink": ["yiming"], - "client-library": ["tao"], + "doris-sink-json": ["xinhao"], + "starrocks-sink-json": ["xinhao"], + "deltalake-sink-json": ["xinhao"], + "pinot-sink-json": ["yiming"], + "client-library-none": ["tao"], } def get_failed_tests(get_test_status, test_map): diff --git a/ci/scripts/pr.env.sh b/ci/scripts/pr.env.sh old mode 100644 new mode 100755 index dff62c33383a2..84fe372d5d574 --- a/ci/scripts/pr.env.sh +++ b/ci/scripts/pr.env.sh @@ -6,6 +6,4 @@ set -euo pipefail # Don't run e2e compaction test in PR build export RUN_COMPACTION=0; # Don't run meta store backup/recovery test -export RUN_META_BACKUP=0; -# Don't run delete-range random test -export RUN_DELETE_RANGE=0; \ No newline at end of file +export RUN_META_BACKUP=0; \ No newline at end of file diff --git a/ci/scripts/regress-test.sh b/ci/scripts/regress-test.sh index aa5912e591df8..384c321afe5eb 100755 --- a/ci/scripts/regress-test.sh +++ b/ci/scripts/regress-test.sh @@ -40,6 +40,13 @@ dpkg-reconfigure --frontend=noninteractive locales # All the above is required because otherwise psql would throw some warning # that goes into the output file and thus diverges from the expected output file. export PGPASSWORD='postgres'; + +# Load extensions. This shall only be done once per database, so not part of test runner. +psql -h db -p 5432 -d postgres -U postgres \ + -c 'create extension pgcrypto;' \ + -c 'create extension hstore;' \ + -c 'create extension tablefunc;' + RUST_BACKTRACE=1 target/debug/risingwave_regress_test --host db \ -p 5432 \ -u postgres \ diff --git a/ci/scripts/release.sh b/ci/scripts/release.sh index 32d9b801a7e26..e35459b71495b 100755 --- a/ci/scripts/release.sh +++ b/ci/scripts/release.sh @@ -4,6 +4,7 @@ set -euo pipefail REPO_ROOT=${PWD} +ARCH="$(uname -m)" echo "--- Check env" if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhook" ] && [[ -z "${BINARY_NAME+x}" ]]; then @@ -11,22 +12,22 @@ if [ "${BUILDKITE_SOURCE}" != "schedule" ] && [ "${BUILDKITE_SOURCE}" != "webhoo fi echo "--- Install aws cli" -curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +curl "https://awscli.amazonaws.com/awscli-exe-linux-${ARCH}.zip" -o "awscliv2.zip" unzip -q awscliv2.zip && ./aws/install && mv /usr/local/bin/aws /bin/aws echo "--- Install lld" # The lld in the CentOS 7 repository is too old and contains a bug that causes a linker error. # So we install a newer version here. (17.0.6, latest version at the time of writing) # It is manually built in the same environent and uploaded to S3. -aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_x86_64.tar.gz . -tar -zxvf llvm-lld-manylinux2014_x86_64.tar.gz --directory=/usr/local +aws s3 cp s3://ci-deps-dist/llvm-lld-manylinux2014_${ARCH}.tar.gz . +tar -zxvf llvm-lld-manylinux2014_${ARCH}.tar.gz --directory=/usr/local ld.lld --version echo "--- Install dependencies for openssl" yum install -y perl-core echo "--- Install java and maven" -yum install -y java-11-openjdk java-11-openjdk-devel wget python3 cyrus-sasl-devel +yum install -y java-11-openjdk java-11-openjdk-devel wget python3 python3-devel cyrus-sasl-devel pip3 install toml-cli wget https://ci-deps-dist.s3.amazonaws.com/apache-maven-3.9.3-bin.tar.gz && tar -zxvf apache-maven-3.9.3-bin.tar.gz export PATH="${REPO_ROOT}/apache-maven-3.9.3/bin:$PATH" @@ -40,8 +41,13 @@ source ci/scripts/common.sh unset RUSTC_WRAPPER # disable sccache echo "--- Install protoc3" -curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-x86_64.zip -unzip -o protoc-3.15.8-linux-x86_64.zip -d protoc +PROTOC_ARCH=${ARCH} +if [ ${ARCH} == "aarch64" ]; then + # shellcheck disable=SC1068 + PROTOC_ARCH="aarch_64" +fi +curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v3.15.8/protoc-3.15.8-linux-${PROTOC_ARCH}.zip +unzip -o protoc-3.15.8-linux-${PROTOC_ARCH}.zip -d protoc mv ./protoc/bin/protoc /usr/local/bin/ mv ./protoc/include/* /usr/local/include/ @@ -58,16 +64,16 @@ fi echo "--- Build risingwave release binary" cargo build -p risingwave_cmd_all --features "rw-static-link" --profile release -cargo build --bin risectl --features "rw-static-link" --profile release +cargo build -p risingwave_cmd --bin risectl --features "rw-static-link" --profile release cd target/release && chmod +x risingwave risectl echo "--- Upload nightly binary to s3" if [ "${BUILDKITE_SOURCE}" == "schedule" ]; then - tar -czvf risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz risingwave - aws s3 cp risingwave-"$(date '+%Y%m%d')"-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary + tar -czvf risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz risingwave + aws s3 cp risingwave-"$(date '+%Y%m%d')"-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary elif [[ -n "${BINARY_NAME+x}" ]]; then - tar -czvf risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz risingwave - aws s3 cp risingwave-${BINARY_NAME}-x86_64-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary + tar -czvf risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz risingwave + aws s3 cp risingwave-${BINARY_NAME}-${ARCH}-unknown-linux.tar.gz s3://risingwave-nightly-pre-built-binary fi echo "--- Build connector node" @@ -88,23 +94,31 @@ if [[ -n "${BUILDKITE_TAG}" ]]; then dnf install -y gh echo "--- Release create" - gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p + set +e + response=$(gh api repos/risingwavelabs/risingwave/releases/tags/${BUILDKITE_TAG} 2>&1) + set -euo pipefail + if [[ $response == *"Not Found"* ]]; then + echo "Tag ${BUILDKITE_TAG} does not exist. Creating release..." + gh release create "${BUILDKITE_TAG}" --notes "release ${BUILDKITE_TAG}" -d -p + else + echo "Tag ${BUILDKITE_TAG} already exists. Skipping release creation." + fi echo "--- Release upload risingwave asset" - tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risingwave - gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz + tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz risingwave + gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz echo "--- Release upload risingwave debug info" - tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.dwp.tar.gz risingwave.dwp - gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux.dwp.tar.gz + tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.dwp.tar.gz risingwave.dwp + gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.dwp.tar.gz echo "--- Release upload risectl asset" - tar -czvf risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz risectl - gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-x86_64-unknown-linux.tar.gz + tar -czvf risectl-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz risectl + gh release upload "${BUILDKITE_TAG}" risectl-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux.tar.gz echo "--- Release upload risingwave-all-in-one asset" - tar -czvf risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz risingwave libs - gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-x86_64-unknown-linux-all-in-one.tar.gz + tar -czvf risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux-all-in-one.tar.gz risingwave libs + gh release upload "${BUILDKITE_TAG}" risingwave-"${BUILDKITE_TAG}"-${ARCH}-unknown-linux-all-in-one.tar.gz fi diff --git a/ci/scripts/run-backfill-tests.sh b/ci/scripts/run-backfill-tests.sh index 46bad790d21f7..a61819368c685 100755 --- a/ci/scripts/run-backfill-tests.sh +++ b/ci/scripts/run-backfill-tests.sh @@ -4,7 +4,7 @@ # USAGE: # ```sh -# ./ci/scripts/run-backfill-tests.sh +# profile=(ci-release|ci-dev) ./ci/scripts/run-backfill-tests.sh # ``` # Example progress: # dev=> select * from rw_catalog.rw_ddl_progress; @@ -23,7 +23,17 @@ BACKGROUND_DDL_DIR=$TEST_DIR/background_ddl COMMON_DIR=$BACKGROUND_DDL_DIR/common CLUSTER_PROFILE='ci-1cn-1fe-kafka-with-recovery' -export RUST_LOG="info,risingwave_meta::barrier::progress=debug,risingwave_meta::rpc::ddl_controller=debug" +echo "--- Configuring cluster profiles" +if [[ -n "${BUILDKITE:-}" ]]; then + echo "Running in buildkite" + RUNTIME_CLUSTER_PROFILE='ci-3cn-1fe' + MINIO_RATE_LIMIT_CLUSTER_PROFILE='ci-3cn-1fe-with-minio-rate-limit' +else + echo "Running locally" + RUNTIME_CLUSTER_PROFILE='ci-3cn-1fe-with-monitoring' + MINIO_RATE_LIMIT_CLUSTER_PROFILE='ci-3cn-1fe-with-monitoring-and-minio-rate-limit' +fi +export RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ run_sql_file() { psql -h localhost -p 4566 -d dev -U root -f "$@" @@ -55,8 +65,8 @@ rename_logs_with_prefix() { } kill_cluster() { - cargo make kill - cargo make wait-processes-exit + cargo make ci-kill-no-dump-logs + wait } restart_cluster() { @@ -145,7 +155,6 @@ test_backfill_tombstone() { ./risedev psql -c "CREATE MATERIALIZED VIEW m1 as select * from tomb;" echo "--- Kill cluster" kill_cluster - cargo make wait-processes-exit wait } @@ -166,9 +175,7 @@ test_replication_with_column_pruning() { run_sql_file "$PARENT_PATH"/sql/backfill/replication_with_column_pruning/select.sql &1 1>out.log & + echo "[INFO] Upstream is ingesting in background" + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt' + + wait + + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_arrangement.slt' + + cargo make ci-kill +} + +test_no_shuffle_backfill_snapshot_and_upstream_runtime() { + echo "--- e2e, test_no_shuffle_backfill_snapshot_and_upstream_runtime, $RUNTIME_CLUSTER_PROFILE" + cargo make ci-start $RUNTIME_CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert_snapshot.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert_upstream.slt' 2>&1 1>out.log & + echo "[INFO] Upstream is ingesting in background" + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt' + wait + + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_no_shuffle.slt' + + kill_cluster +} + +test_backfill_snapshot_runtime() { + echo "--- e2e, test_backfill_snapshot_runtime, $RUNTIME_CLUSTER_PROFILE" + cargo make ci-start $RUNTIME_CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert_snapshot.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_no_shuffle.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_arrangement.slt' + + kill_cluster +} + +# Throttle the storage throughput. +# Arrangement Backfill should not fail because of this. +test_backfill_snapshot_with_limited_storage_throughput() { + echo "--- e2e, test_backfill_snapshot_with_limited_storage_throughput, $MINIO_RATE_LIMIT_CLUSTER_PROFILE" + cargo make ci-start $MINIO_RATE_LIMIT_CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert_snapshot.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_no_shuffle.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_arrangement.slt' + + kill_cluster +} + +# Test case where we do backfill with PK of 10 columns to measure performance impact. +test_backfill_snapshot_with_wider_rows() { + echo "--- e2e, test_backfill_snapshot_with_wider_rows, $RUNTIME_CLUSTER_PROFILE" + cargo make ci-start $RUNTIME_CLUSTER_PROFILE + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_wide_table.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/insert_wide_snapshot.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/create_no_shuffle_mv.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_no_shuffle.slt' + sqllogictest -p 4566 -d dev 'e2e_test/backfill/runtime/validate_rows_arrangement.slt' + + kill_cluster } main() { @@ -206,6 +285,23 @@ main() { test_backfill_tombstone test_replication_with_column_pruning test_sink_backfill_recovery + + # Only if profile is "ci-release", run it. + if [[ ${profile:-} == "ci-release" ]]; then + echo "--- Using release profile, running backfill performance tests." + # Need separate tests, we don't want to backfill concurrently. + # It's difficult to measure the time taken for each backfill if we do so. + test_no_shuffle_backfill_snapshot_and_upstream_runtime + test_arrangement_backfill_snapshot_and_upstream_runtime + + # Backfill will happen in sequence here. + test_backfill_snapshot_runtime + test_backfill_snapshot_with_wider_rows + test_backfill_snapshot_with_limited_storage_throughput + + # No upstream only tests, because if there's no snapshot, + # Backfill will complete almost immediately. + fi } main diff --git a/ci/scripts/run-e2e-test.sh b/ci/scripts/run-e2e-test.sh index 86eae7a146f5f..24eaa3881c123 100755 --- a/ci/scripts/run-e2e-test.sh +++ b/ci/scripts/run-e2e-test.sh @@ -26,6 +26,10 @@ if [[ $mode == "standalone" ]]; then source ci/scripts/standalone-utils.sh fi +if [[ $mode == "single-node" ]]; then + source ci/scripts/single-node-utils.sh +fi + cluster_start() { if [[ $mode == "standalone" ]]; then mkdir -p "$PREFIX_LOG" @@ -33,6 +37,13 @@ cluster_start() { cargo make pre-start-dev start_standalone "$PREFIX_LOG"/standalone.log & cargo make dev standalone-minio-etcd + elif [[ $mode == "single-node" ]]; then + mkdir -p "$PREFIX_LOG" + cargo make clean-data + cargo make pre-start-dev + start_single_node "$PREFIX_LOG"/single-node.log & + # Give it a while to make sure the single-node is ready. + sleep 3 else cargo make ci-start "$mode" fi @@ -44,6 +55,9 @@ cluster_stop() { stop_standalone # Don't check standalone logs, they will exceed the limit. cargo make kill + elif [[ $mode == "single-node" ]] + then + stop_single_node else cargo make ci-kill fi @@ -76,31 +90,35 @@ cluster_start sqllogictest -p 4566 -d dev './e2e_test/ddl/**/*.slt' --junit "batch-ddl-${profile}" sqllogictest -p 4566 -d dev './e2e_test/background_ddl/basic.slt' --junit "batch-ddl-${profile}" sqllogictest -p 4566 -d dev './e2e_test/visibility_mode/*.slt' --junit "batch-${profile}" +sqllogictest -p 4566 -d dev './e2e_test/ttl/ttl.slt' sqllogictest -p 4566 -d dev './e2e_test/database/prepare.slt' sqllogictest -p 4566 -d test './e2e_test/database/test.slt' echo "--- e2e, $mode, Apache Superset" sqllogictest -p 4566 -d dev './e2e_test/superset/*.slt' --junit "batch-${profile}" -echo "--- e2e, $mode, python udf" +echo "--- e2e, $mode, external python udf" python3 e2e_test/udf/test.py & sleep 1 -sqllogictest -p 4566 -d dev './e2e_test/udf/udf.slt' +sqllogictest -p 4566 -d dev './e2e_test/udf/external_udf.slt' pkill python3 sqllogictest -p 4566 -d dev './e2e_test/udf/alter_function.slt' sqllogictest -p 4566 -d dev './e2e_test/udf/graceful_shutdown_python.slt' +sqllogictest -p 4566 -d dev './e2e_test/udf/always_retry_python.slt' # FIXME: flaky test # sqllogictest -p 4566 -d dev './e2e_test/udf/retry_python.slt' -echo "--- e2e, $mode, java udf" +echo "--- e2e, $mode, external java udf" java -jar risingwave-udf-example.jar & sleep 1 -sqllogictest -p 4566 -d dev './e2e_test/udf/udf.slt' +sqllogictest -p 4566 -d dev './e2e_test/udf/external_udf.slt' pkill java -echo "--- e2e, $mode, wasm udf" +echo "--- e2e, $mode, embedded udf" sqllogictest -p 4566 -d dev './e2e_test/udf/wasm_udf.slt' +sqllogictest -p 4566 -d dev './e2e_test/udf/js_udf.slt' +sqllogictest -p 4566 -d dev './e2e_test/udf/python_udf.slt' echo "--- Kill cluster" cluster_stop @@ -133,22 +151,6 @@ RUST_BACKTRACE=1 target/debug/risingwave_e2e_extended_mode_test --host 127.0.0.1 echo "--- Kill cluster" cluster_stop -if [[ "$RUN_DELETE_RANGE" -eq "1" ]]; then - echo "--- e2e, ci-delete-range-test" - cargo make clean-data - RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ - cargo make ci-start ci-delete-range-test - download-and-decompress-artifact delete-range-test-"$profile" target/debug/ - mv target/debug/delete-range-test-"$profile" target/debug/delete-range-test - chmod +x ./target/debug/delete-range-test - - config_path=".risingwave/config/risingwave.toml" - ./target/debug/delete-range-test --ci-mode --state-store hummock+minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001 --config-path "${config_path}" - - echo "--- Kill cluster" - cluster_stop -fi - if [[ "$RUN_COMPACTION" -eq "1" ]]; then echo "--- e2e, ci-compaction-test, nexmark_q7" RUST_LOG="info,risingwave_stream=info,risingwave_batch=info,risingwave_storage=info" \ diff --git a/ci/scripts/run-unit-test.sh b/ci/scripts/run-unit-test.sh index 6f2093060f370..8fe4b150bf866 100755 --- a/ci/scripts/run-unit-test.sh +++ b/ci/scripts/run-unit-test.sh @@ -10,10 +10,6 @@ cd ${REPO_ROOT}/src/expr/udf/python python3 -m pytest cd ${REPO_ROOT} -echo "+++ Run unit tests with coverage" +echo "+++ Run unit tests" # use tee to disable progress bar -NEXTEST_PROFILE=ci cargo llvm-cov nextest --lcov --output-path lcov.info --features failpoints,sync_point --workspace --exclude risingwave_simulation - -echo "--- Codecov upload coverage reports" -curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov -./codecov -t "$CODECOV_TOKEN" -s . -F rust +NEXTEST_PROFILE=ci cargo nextest run --features failpoints,sync_point --workspace --exclude risingwave_simulation diff --git a/ci/scripts/single-node-utils.sh b/ci/scripts/single-node-utils.sh new file mode 100755 index 0000000000000..8e75232173703 --- /dev/null +++ b/ci/scripts/single-node-utils.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash + +set -euo pipefail + +export RW_PREFIX=$PWD/.risingwave +export PREFIX_BIN=./target/debug +export PREFIX_LOG=$RW_PREFIX/log + +# You can fill up this section by consulting +# .risingwave/log/risedev.log, after calling ./risedev d full. +# It is expected that minio, etcd will be started after this is called. +start_single_node() { + mkdir -p "$HOME/.risingwave/state_store" + mkdir -p "$HOME/.risingwave/meta_store" + RUST_BACKTRACE=1 "$PREFIX_BIN"/risingwave >"$1" 2>&1 +} + +stop_single_node() { + pkill risingwave + rm -rf "$HOME/.risingwave/state_store" + rm -rf "$HOME/.risingwave/meta_store" +} + +wait_single_node() { + set +e + timeout 20s bash -c ' + while true; do + echo "Polling every 1s for single_node to be ready for 20s" + if psql -h localhost -p 4566 -d dev -U root -c "SELECT 1;" 1000000 AND 0.908 * price < 50000000 diff --git a/ci/workflows/docker.yml b/ci/workflows/docker.yml index d97d99af7691d..40f0fe290d921 100644 --- a/ci/workflows/docker.yml +++ b/ci/workflows/docker.yml @@ -46,14 +46,14 @@ steps: DOCKER_TOKEN: docker-token retry: *auto-retry - - label: "pre build binary" + - label: "pre build binary: amd64" command: "ci/scripts/release.sh" plugins: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: - run: release-env + - docker-compose#v5.1.0: + run: release-env-x86 config: ci/docker-compose.yml mount-buildkite-agent: true propagate-environment: true @@ -61,3 +61,21 @@ steps: - BINARY_NAME - GITHUB_TOKEN retry: *auto-retry + + - label: "pre build binary: aarch64 " + command: "ci/scripts/release.sh" + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + GITHUB_TOKEN: github-token + - docker-compose#v5.1.0: + run: release-env-arm + config: ci/docker-compose.yml + mount-buildkite-agent: true + propagate-environment: true + environment: + - BINARY_NAME + - GITHUB_TOKEN + agents: + queue: "linux-arm64" + retry: *auto-retry diff --git a/ci/workflows/gen-flamegraph-cron.yml b/ci/workflows/gen-flamegraph-cron.yml index ff78f04e3f623..3ccd9eed4f44d 100644 --- a/ci/workflows/gen-flamegraph-cron.yml +++ b/ci/workflows/gen-flamegraph-cron.yml @@ -7,7 +7,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -22,7 +22,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: ci-flamegraph-env config: ci/docker-compose.yml mount-buildkite-agent: true diff --git a/ci/workflows/gen-flamegraph.yml b/ci/workflows/gen-flamegraph.yml index 9d4140d5a7852..e129f95f92a61 100644 --- a/ci/workflows/gen-flamegraph.yml +++ b/ci/workflows/gen-flamegraph.yml @@ -8,7 +8,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -26,7 +26,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: ci-flamegraph-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -48,7 +48,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: ci-flamegraph-env config: ci/docker-compose.yml mount-buildkite-agent: true diff --git a/ci/workflows/main-cron.yml b/ci/workflows/main-cron.yml index f2806a458ee53..84ea38fe43cee 100644 --- a/ci/workflows/main-cron.yml +++ b/ci/workflows/main-cron.yml @@ -13,11 +13,11 @@ steps: || build.env("CI_STEPS") =~ /(^|,)build(,|$$)/ key: "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - timeout_in_minutes: 30 + timeout_in_minutes: 20 retry: *auto-retry - label: "build other components" @@ -31,7 +31,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -40,7 +40,7 @@ steps: timeout_in_minutes: 12 retry: *auto-retry - - label: "build (deterministic simulation)" + - label: "build simulation test" command: "ci/scripts/build-simulation.sh" if: | !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null @@ -48,7 +48,7 @@ steps: || build.env("CI_STEPS") =~ /(^|,)build-simulation(,|$$)/ key: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -63,7 +63,7 @@ steps: || build.env("CI_STEPS") =~ /(^|,)docslt(,|$$)/ key: "docslt" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -82,7 +82,7 @@ steps: - "build-other" - "docslt" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -104,7 +104,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-sqllogictest-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -126,7 +126,7 @@ steps: - "build" - "docslt" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -145,7 +145,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: source-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -164,7 +164,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -184,7 +184,7 @@ steps: - "build-simulation" plugins: - ./ci/plugins/swapfile - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -206,7 +206,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: CODECOV_TOKEN: my-codecov-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml environment: @@ -214,7 +214,7 @@ steps: timeout_in_minutes: 25 retry: *auto-retry - - label: "unit test (deterministic simulation)" + - label: "unit test (madsim)" key: "unit-test-deterministic" command: "MADSIM_TEST_NUM=100 timeout 15m ci/scripts/deterministic-unit-test.sh" if: | @@ -222,14 +222,14 @@ steps: || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)unit-tests?-deterministic-simulation(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true timeout_in_minutes: 15 retry: *auto-retry - - label: "integration test (deterministic simulation) - scale" + - label: "integration test (madsim) - scale" key: "integration-test-deterministic-scale" command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh scale::" if: | @@ -238,7 +238,7 @@ steps: || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -246,7 +246,7 @@ steps: timeout_in_minutes: 70 retry: *auto-retry - - label: "integration test (deterministic simulation) - recovery" + - label: "integration test (madsim) - recovery" key: "integration-test-deterministic-recovery" command: "TEST_NUM=60 ci/scripts/deterministic-it-test.sh recovery::" if: | @@ -255,7 +255,7 @@ steps: || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -263,16 +263,16 @@ steps: timeout_in_minutes: 70 retry: *auto-retry - - label: "integration test (deterministic simulation) - others" - key: "integration-test-deterministic-others" - command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh backfill_tests:: storage:: sink::" + - label: "integration test (madsim) - backfill" + key: "integration-test-deterministic-backfill" + command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh backfill_tests::" if: | !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -280,7 +280,41 @@ steps: timeout_in_minutes: 70 retry: *auto-retry - - label: "end-to-end test (deterministic simulation)" + - label: "integration test (madsim) - storage" + key: "integration-test-deterministic-storage" + command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh storage::" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" + || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ + depends_on: "build-simulation" + plugins: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 70 + retry: *auto-retry + + - label: "integration test (madsim) - sink" + key: "integration-test-deterministic-sink" + command: "TEST_NUM=30 ci/scripts/deterministic-it-test.sh sink::" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-integration-test-deterministic-simulation" + || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ + depends_on: "build-simulation" + plugins: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 70 + retry: *auto-retry + + - label: "end-to-end test (madsim)" key: "e2e-test-deterministic" command: "TEST_NUM=64 timeout 55m ci/scripts/deterministic-e2e-test.sh" if: | @@ -292,7 +326,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -302,7 +336,7 @@ steps: timeout_in_minutes: 60 retry: *auto-retry - - label: "recovery test (deterministic simulation)" + - label: "recovery test (madsim)" key: "recovery-test-deterministic" command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.0 timeout 55m ci/scripts/deterministic-recovery-test.sh" if: | @@ -311,16 +345,36 @@ steps: || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs + # Only upload zipped files, otherwise the logs is too much. + - ./ci/plugins/upload-failure-logs-zipped timeout_in_minutes: 60 retry: *auto-retry # Ddl statements will randomly run with background_ddl. - - label: "background_ddl recovery test (deterministic simulation)" + - label: "background_ddl, arrangement_backfill recovery test (madsim)" + key: "background-ddl-arrangement-backfill-recovery-test-deterministic" + command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 USE_ARRANGEMENT_BACKFILL=--use-arrangement-backfill timeout 55m ci/scripts/deterministic-recovery-test.sh" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation" + || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/ + depends_on: "build-simulation" + plugins: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + # Only upload zipped files, otherwise the logs is too much. + - ./ci/plugins/upload-failure-logs-zipped + timeout_in_minutes: 60 + retry: *auto-retry + + # Ddl statements will randomly run with background_ddl. + - label: "background_ddl recovery test (madsim)" key: "background-ddl-recovery-test-deterministic" command: "TEST_NUM=12 KILL_RATE=1.0 BACKGROUND_DDL_RATE=0.8 timeout 55m ci/scripts/deterministic-recovery-test.sh" if: | @@ -329,11 +383,12 @@ steps: || build.env("CI_STEPS") =~ /(^|,)recovery-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - - ./ci/plugins/upload-failure-logs + # Only upload zipped files, otherwise the logs is too much. + - ./ci/plugins/upload-failure-logs-zipped timeout_in_minutes: 60 retry: *auto-retry @@ -349,12 +404,12 @@ steps: - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 5 + timeout_in_minutes: 7 retry: *auto-retry - label: "end-to-end iceberg sink v2 test (release)" @@ -368,12 +423,12 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 + timeout_in_minutes: 11 retry: *auto-retry - label: "e2e java-binding test (release)" @@ -387,7 +442,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -409,7 +464,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: S3_SOURCE_TEST_CONF: ci_s3_source_test_aws - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -431,7 +486,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: S3_SOURCE_TEST_CONF: ci_s3_source_test_aws - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -453,7 +508,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: S3_SOURCE_TEST_CONF: ci_s3_source_test_aws - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -475,7 +530,29 @@ steps: - seek-oss/aws-sm#v2.3.1: env: S3_SOURCE_TEST_CONF: ci_s3_source_test_aws - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + environment: + - S3_SOURCE_TEST_CONF + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 25 + retry: *auto-retry + + - label: "S3_v2 source batch read on AWS (json parser)" + key: "s3-v2-source-batch-read-check-aws-json-parser" + command: "ci/scripts/s3-source-test.sh -p ci-release -s 'fs_source_batch.py json'" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-s3-source-tests" + || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/ + depends_on: build + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + S3_SOURCE_TEST_CONF: ci_s3_source_test_aws + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -497,7 +574,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: S3_SOURCE_TEST_CONF: ci_s3_source_test_aws - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -516,7 +593,7 @@ steps: || build.env("CI_STEPS") =~ /(^|,)s3-source-tests?(,|$$)/ depends_on: build plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -536,7 +613,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: S3_SOURCE_TEST_CONF: ci_s3_source_test_aws - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -559,7 +636,7 @@ steps: # - seek-oss/aws-sm#v2.3.1: # env: # S3_SOURCE_TEST_CONF: ci_s3_source_test_aws - # - docker-compose#v4.9.0: + # - docker-compose#v5.1.0: # run: rw-build-env # config: ci/docker-compose.yml # mount-buildkite-agent: true @@ -584,7 +661,7 @@ steps: env: ASTRA_STREAMING_TEST_TOKEN: astra_streaming_test_token STREAMNATIVE_CLOUD_TEST_CONF: streamnative_cloud_test_conf - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -602,7 +679,7 @@ steps: || build.pull_request.labels includes "ci/run-micro-benchmarks" || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -624,7 +701,7 @@ steps: env: BUILDKITE_TOKEN: buildkite_token GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -644,7 +721,7 @@ steps: depends_on: - "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -678,7 +755,7 @@ steps: depends_on: - "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -686,7 +763,7 @@ steps: - label: "Backfill tests" key: "backfill-tests" - command: "ci/scripts/backfill-test.sh -p ci-release" + command: "BUILDKITE=${BUILDKITE:-} ci/scripts/backfill-test.sh -p ci-release" if: | !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-backfill-tests" @@ -694,12 +771,12 @@ steps: depends_on: - "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 20 + timeout_in_minutes: 22 retry: *auto-retry - label: "e2e standalone binary test" @@ -713,6 +790,26 @@ steps: - "build" - "build-other" - "docslt" + plugins: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 25 + retry: *auto-retry + + - label: "e2e single-node binary test" + key: "e2e-single-node-binary-tests" + command: "ci/scripts/e2e-test.sh -p ci-release -m single-node" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-e2e-single-node-tests" + || build.env("CI_STEPS") =~ /(^|,)e2e-single-node-tests?(,|$$)/ + depends_on: + - "build" + - "build-other" + - "docslt" plugins: - docker-compose#v4.9.0: run: rw-build-env @@ -733,7 +830,7 @@ steps: - "build" - "docslt" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -752,7 +849,96 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + + - label: "end-to-end redis sink test" + key: "e2e-redis-sink-tests" + command: "ci/scripts/e2e-redis-sink-test.sh -p ci-release" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-e2e-redis-sink-tests" + || build.env("CI_STEPS") =~ /(^|,)e2e-redis-sink-tests?(,|$$)/ + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + + # Causes ci error, close it first, fix it later + # - label: "set vm_max_map_count_2000000" + # key: "set-vm_max_map_count" + # if: | + # !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + # || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" + # || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/ + # command: "sudo sysctl -w vm.max_map_count=2000000" + # depends_on: + # - "build" + # - "build-other" + + # - label: "end-to-end doris sink test" + # key: "e2e-doris-sink-tests" + # command: "ci/scripts/e2e-doris-sink-test.sh -p ci-release" + # if: | + # !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + # || build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" + # || build.env("CI_STEPS") =~ /(^|,)e2e-doris-sink-tests?(,|$$)/ + # depends_on: + # - "build" + # - "build-other" + # - "set-vm_max_map_count" + # plugins: + # - docker-compose#v5.1.0: + # run: sink-doris-env + # config: ci/docker-compose.yml + # mount-buildkite-agent: true + # - ./ci/plugins/upload-failure-logs + # timeout_in_minutes: 10 + # retry: *auto-retry + + - label: "end-to-end starrocks sink test" + key: "e2e-starrocks-sink-tests" + command: "ci/scripts/e2e-starrocks-sink-test.sh -p ci-release" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" + || build.env("CI_STEPS") =~ /(^|,)e2e-starrocks-sink-tests?(,|$$)/ + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + + - label: "end-to-end cassandra sink test" + key: "e2e-cassandra-sink-tests" + command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-release" + if: | + !(build.pull_request.labels includes "ci/main-cron/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests" + || build.env("CI_STEPS") =~ /(^|,)e2e-cassandra-sink-tests?(,|$$)/ + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -771,7 +957,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -790,7 +976,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -809,7 +995,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -822,15 +1008,15 @@ steps: timeout_in_minutes: 10 retry: *auto-retry - - label: "release" + - label: "release amd64" command: "ci/scripts/release.sh" if: build.tag != null plugins: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: - run: release-env + - docker-compose#v5.1.0: + run: release-env-x86 config: ci/docker-compose.yml mount-buildkite-agent: true environment: @@ -840,6 +1026,26 @@ steps: timeout_in_minutes: 60 retry: *auto-retry + - label: "release aarch64" + command: "ci/scripts/release.sh" + if: build.tag != null + plugins: + - seek-oss/aws-sm#v2.3.1: + env: + GITHUB_TOKEN: github-token + - docker-compose#v5.1.0: + run: release-env-arm + config: ci/docker-compose.yml + mount-buildkite-agent: true + environment: + - GITHUB_TOKEN + - BUILDKITE_TAG + - BUILDKITE_SOURCE + agents: + queue: "linux-arm64" + timeout_in_minutes: 60 + retry: *auto-retry + - label: "release docker image: amd64" command: "ci/scripts/docker.sh" key: "build-amd64" diff --git a/ci/workflows/pull-request.yml b/ci/workflows/pull-request.yml index e6b4e3ea12571..12fee00c34a8c 100644 --- a/ci/workflows/pull-request.yml +++ b/ci/workflows/pull-request.yml @@ -24,7 +24,7 @@ steps: || build.pull_request.labels includes "ci/run-build" || build.env("CI_STEPS") =~ /(^|,)build(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -42,7 +42,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -59,7 +59,7 @@ steps: || build.pull_request.labels includes "ci/run-build-simulation" || build.env("CI_STEPS") =~ /(^|,)build-simulation(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -73,7 +73,7 @@ steps: || build.pull_request.labels includes "ci/run-docslt" || build.env("CI_STEPS") =~ /(^|,)docslt(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -91,12 +91,12 @@ steps: - "build-other" - "docslt" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 15 + timeout_in_minutes: 19 retry: *auto-retry - label: "end-to-end test (parallel)" @@ -109,7 +109,7 @@ steps: - "build" - "docslt" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -124,7 +124,7 @@ steps: - "build" - "docslt" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -137,7 +137,7 @@ steps: command: "ci/scripts/e2e-test-parallel-in-memory.sh -p ci-dev" depends_on: "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -155,7 +155,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: source-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -173,7 +173,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -189,7 +189,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -209,12 +209,12 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 5 + timeout_in_minutes: 10 retry: *auto-retry - label: "end-to-end iceberg sink v2 test" @@ -224,12 +224,12 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 + timeout_in_minutes: 15 retry: *auto-retry - label: "end-to-end iceberg cdc test" @@ -239,12 +239,12 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true - ./ci/plugins/upload-failure-logs - timeout_in_minutes: 10 + timeout_in_minutes: 15 retry: *auto-retry - label: "end-to-end pulsar sink test" @@ -254,7 +254,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -269,7 +269,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -284,7 +284,77 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + + - label: "end-to-end redis sink test" + if: build.pull_request.labels includes "ci/run-e2e-redis-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-redis-sink-tests?(,|$$)/ + command: "ci/scripts/e2e-redis-sink-test.sh -p ci-dev" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + + # Causes ci error, close it first, fix it later + # - label: "set vm_max_map_count_2000000" + # key: "set-vm_max_map_count" + # if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/ + # command: "sudo sysctl -w vm.max_map_count=2000000" + # depends_on: + # - "build" + # - "build-other" + + # - label: "end-to-end doris sink test" + # if: build.pull_request.labels includes "ci/run-e2e-doris-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-doris-sink-tests?(,|$$)/ + # command: "ci/scripts/e2e-doris-sink-test.sh -p ci-dev" + # depends_on: + # - "build" + # - "build-other" + # - "set-vm_max_map_count" + # plugins: + # - docker-compose#v5.1.0: + # run: sink-doris-env + # config: ci/docker-compose.yml + # mount-buildkite-agent: true + # - ./ci/plugins/upload-failure-logs + # timeout_in_minutes: 10 + # retry: *auto-retry + + - label: "end-to-end starrocks sink test" + if: build.pull_request.labels includes "ci/run-e2e-starrocks-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-starrocks-sink-tests?(,|$$)/ + command: "ci/scripts/e2e-starrocks-sink-test.sh -p ci-dev" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: + run: sink-test-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 10 + retry: *auto-retry + + - label: "end-to-end cassandra sink test" + if: build.pull_request.labels includes "ci/run-e2e-cassandra-sink-tests" || build.env("CI_STEPS") =~ /(^|,) e2e-cassandra-sink-tests?(,|$$)/ + command: "ci/scripts/e2e-cassandra-sink-test.sh -p ci-dev" + depends_on: + - "build" + - "build-other" + plugins: + - docker-compose#v5.1.0: run: sink-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -299,7 +369,7 @@ steps: - "build" - "build-other" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -315,7 +385,7 @@ steps: || build.env("CI_STEPS") =~ /(^|,)regress-tests?(,|$$)/ depends_on: "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: regress-test-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -337,7 +407,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: CODECOV_TOKEN: my-codecov-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml environment: @@ -364,7 +434,32 @@ steps: args: "--no-progress" paths: - ".cargo/advisory-db" - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + timeout_in_minutes: 25 + retry: *auto-retry + + - label: "check dylint" + command: "ci/scripts/check-dylint.sh" + if: | + !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null + || build.pull_request.labels includes "ci/run-check" + || build.env("CI_STEPS") =~ /(^|,)check(,|$$)/ + plugins: + - gencer/cache#v2.4.10: + id: cache + key: "v1-cache-{{ id }}-{{ runner.os }}-{{ checksum 'Cargo.lock' }}" + restore-keys: + - "v1-cache-{{ id }}-{{ runner.os }}-" + - "v1-cache-{{ id }}-" + backend: s3 + s3: + bucket: ci-cache-bucket + args: "--no-progress" + paths: + - ".cargo/advisory-db" + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml timeout_in_minutes: 25 @@ -377,7 +472,7 @@ steps: || build.pull_request.labels includes "ci/run-unit-test-deterministic-simulation" || build.env("CI_STEPS") =~ /(^|,)unit-tests?-deterministic-simulation(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -393,7 +488,7 @@ steps: || build.env("CI_STEPS") =~ /(^|,)integration-tests?-deterministic-simulation(,|$$)/ depends_on: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -412,7 +507,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -424,7 +519,7 @@ steps: retry: *auto-retry - label: "recovery test (deterministic simulation)" - command: "TEST_NUM=8 KILL_RATE=0.5 BACKGROUND_DDL_RATE=0.0 ci/scripts/deterministic-recovery-test.sh" + command: "TEST_NUM=8 KILL_RATE=0.4 BACKGROUND_DDL_RATE=0.0 ci/scripts/deterministic-recovery-test.sh" if: | !(build.pull_request.labels includes "ci/skip-ci") && build.env("CI_STEPS") == null || build.pull_request.labels includes "ci/run-recovery-test-deterministic-simulation" @@ -434,14 +529,15 @@ steps: # - seek-oss/aws-sm#v2.3.1: # env: # BUILDKITE_ANALYTICS_TOKEN: buildkite-build-analytics-deterministic-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true + # Only upload zipped files, otherwise the logs is too much. + - ./ci/plugins/upload-failure-logs-zipped # - test-collector#v1.0.0: # files: "*-junit.xml" # format: "junit" - - ./ci/plugins/upload-failure-logs timeout_in_minutes: 25 cancel_on_build_failing: true retry: *auto-retry @@ -453,7 +549,7 @@ steps: || build.pull_request.labels includes "ci/run-misc-check" || build.env("CI_STEPS") =~ /(^|,)misc-check(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml - shellcheck#v1.2.0: @@ -477,7 +573,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -494,7 +590,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: ci-flamegraph-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -517,7 +613,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: ci-flamegraph-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -539,7 +635,7 @@ steps: depends_on: - "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -568,19 +664,19 @@ steps: depends_on: - "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: ci-flamegraph-env config: ci/docker-compose.yml mount-buildkite-agent: true timeout_in_minutes: 40 - label: "Backfill tests" - command: "ci/scripts/backfill-test.sh -p ci-dev" + command: "BUILDKITE=${BUILDKITE:-} ci/scripts/backfill-test.sh -p ci-dev" if: build.pull_request.labels includes "ci/run-backfill-tests" || build.env("CI_STEPS") =~ /(^|,)backfill-tests?(,|$$)/ depends_on: - "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -590,6 +686,22 @@ steps: - label: "e2e standalone binary test" command: "ci/scripts/e2e-test.sh -p ci-dev -m standalone" if: build.pull_request.labels includes "ci/run-e2e-standalone-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-standalone-tests?(,|$$)/ + depends_on: + - "build" + - "build-other" + - "docslt" + plugins: + - docker-compose#v5.1.0: + run: rw-build-env + config: ci/docker-compose.yml + mount-buildkite-agent: true + - ./ci/plugins/upload-failure-logs + timeout_in_minutes: 30 + retry: *auto-retry + + - label: "e2e single-node binary test" + command: "ci/scripts/e2e-test.sh -p ci-dev -m single-node" + if: build.pull_request.labels includes "ci/run-e2e-single-node-tests" || build.env("CI_STEPS") =~ /(^|,)e2e-single-node-tests?(,|$$)/ depends_on: - "build" - "build-other" @@ -612,7 +724,7 @@ steps: - "build-simulation" plugins: - ./ci/plugins/swapfile - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -631,7 +743,7 @@ steps: key: "run-micro-benchmarks" if: build.pull_request.labels includes "ci/run-micro-benchmarks" || build.env("CI_STEPS") =~ /(^|,)micro-benchmarks?(,|$$)/ plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -648,7 +760,7 @@ steps: env: BUILDKITE_TOKEN: buildkite_token GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true diff --git a/ci/workflows/sqlsmith-snapshots.yml b/ci/workflows/sqlsmith-snapshots.yml index dfebb44ed8997..cda8622f7cd63 100644 --- a/ci/workflows/sqlsmith-snapshots.yml +++ b/ci/workflows/sqlsmith-snapshots.yml @@ -4,7 +4,7 @@ steps: command: "ci/scripts/build.sh -p ci-dev" key: "build" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -14,7 +14,7 @@ steps: command: "ci/scripts/build-simulation.sh" key: "build-simulation" plugins: - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true @@ -29,7 +29,7 @@ steps: - seek-oss/aws-sm#v2.3.1: env: GITHUB_TOKEN: github-token - - docker-compose#v4.9.0: + - docker-compose#v5.1.0: run: rw-build-env config: ci/docker-compose.yml mount-buildkite-agent: true diff --git a/clippy.toml b/clippy.toml index bcc3c789ae35a..bbda73f322593 100644 --- a/clippy.toml +++ b/clippy.toml @@ -19,6 +19,7 @@ disallowed-types = [ { path = "num_traits::FromPrimitive", reason = "Please use `From` or `TryFrom` with `OrderedFloat` instead." }, { path = "num_traits::ToPrimitive", reason = "Please use `From` or `TryFrom` with `OrderedFloat` instead." }, { path = "num_traits::NumCast", reason = "Please use `From` or `TryFrom` with `OrderedFloat` instead." }, + { path = "aws_smithy_types::error::display::DisplayErrorContext", reason = "Please use `thiserror_ext::AsReport` instead." }, ] disallowed-macros = [ { path = "lazy_static::lazy_static", reason = "Please use `std::sync::LazyLock` instead." }, diff --git a/dashboard/.node-version b/dashboard/.node-version index 79bdb1b9736cf..df9385826faa2 100644 --- a/dashboard/.node-version +++ b/dashboard/.node-version @@ -1 +1 @@ -v18.7.0 +v20.11.0 \ No newline at end of file diff --git a/dashboard/components/BackPressureTable.tsx b/dashboard/components/BackPressureTable.tsx deleted file mode 100644 index ad790c0cb680a..0000000000000 --- a/dashboard/components/BackPressureTable.tsx +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2024 RisingWave Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ - -import { - Table, - TableCaption, - TableContainer, - Tbody, - Td, - Th, - Thead, - Tr, -} from "@chakra-ui/react" -import { sortBy } from "lodash" -import Head from "next/head" -import { Fragment, useEffect, useState } from "react" -import useErrorToast from "../hook/useErrorToast" -import { - BackPressuresMetrics, - getActorBackPressures, -} from "../pages/api/metric" -import RateBar from "./RateBar" - -export default function BackPressureTable({ - selectedFragmentIds, -}: { - selectedFragmentIds: Set -}) { - const [backPressuresMetrics, setBackPressuresMetrics] = - useState() - const toast = useErrorToast() - - useEffect(() => { - async function doFetch() { - while (true) { - try { - let metrics = await getActorBackPressures() - metrics.outputBufferBlockingDuration = sortBy( - metrics.outputBufferBlockingDuration, - (m) => (m.metric.fragment_id, m.metric.downstream_fragment_id) - ) - setBackPressuresMetrics(metrics) - await new Promise((resolve) => setTimeout(resolve, 5000)) // refresh every 5 secs - } catch (e: any) { - toast(e, "warning") - break - } - } - } - doFetch() - return () => {} - }, [toast]) - - const isSelected = (fragmentId: string) => selectedFragmentIds.has(fragmentId) - - const retVal = ( - - - Back Pressures (Last 30 minutes) - - - - - - - - {backPressuresMetrics && - backPressuresMetrics.outputBufferBlockingDuration - .filter((m) => isSelected(m.metric.fragment_id)) - .map((m) => ( - - - - - ))} - -
Fragment IDs → DownstreamBlock Rate
{`Fragment ${m.metric.fragment_id} -> ${m.metric.downstream_fragment_id}`} - -
- - ) - return ( - - - Streaming Back Pressure - - {retVal} - - ) -} diff --git a/dashboard/components/CatalogModal.tsx b/dashboard/components/CatalogModal.tsx index cf6a2f8cc9e0d..fbc39e2c9fe82 100644 --- a/dashboard/components/CatalogModal.tsx +++ b/dashboard/components/CatalogModal.tsx @@ -32,7 +32,7 @@ import { Relation, relationIsStreamingJob, relationTypeTitleCase, -} from "../pages/api/streaming" +} from "../lib/api/streaming" import { ReactJson } from "./Relations" export function useCatalogModal(relationList: Relation[] | undefined) { diff --git a/dashboard/components/FragmentGraph.tsx b/dashboard/components/FragmentGraph.tsx index 72184d1b2a8bc..6ca2c14dda28e 100644 --- a/dashboard/components/FragmentGraph.tsx +++ b/dashboard/components/FragmentGraph.tsx @@ -479,7 +479,6 @@ export default function FragmentGraph({ - {/* */} ) } diff --git a/dashboard/components/RateBar.tsx b/dashboard/components/RateBar.tsx index dfef2c50b8a57..01f293d2548fe 100644 --- a/dashboard/components/RateBar.tsx +++ b/dashboard/components/RateBar.tsx @@ -17,7 +17,7 @@ import { Box, Text, Tooltip } from "@chakra-ui/react" import { tinycolor } from "@ctrl/tinycolor" -import { p50, p90, p95, p99 } from "../pages/api/metric" +import { p50, p90, p95, p99 } from "../lib/api/metric" import { MetricsSample } from "./metrics" export default function RateBar({ samples }: { samples: MetricsSample[] }) { diff --git a/dashboard/components/RelationDependencyGraph.tsx b/dashboard/components/RelationDependencyGraph.tsx index 0f677101cce17..d2f5052dc368d 100644 --- a/dashboard/components/RelationDependencyGraph.tsx +++ b/dashboard/components/RelationDependencyGraph.tsx @@ -18,6 +18,12 @@ import { theme } from "@chakra-ui/react" import * as d3 from "d3" import { useCallback, useEffect, useRef } from "react" +import { + Relation, + relationIsStreamingJob, + relationType, + relationTypeTitleCase, +} from "../lib/api/streaming" import { Enter, Position, @@ -26,12 +32,6 @@ import { flipLayoutRelation, generateRelationEdges, } from "../lib/layout" -import { - Relation, - relationIsStreamingJob, - relationType, - relationTypeTitleCase, -} from "../pages/api/streaming" import { CatalogModal, useCatalogModal } from "./CatalogModal" function boundBox( diff --git a/dashboard/components/Relations.tsx b/dashboard/components/Relations.tsx index 0422eaa2531fa..6d04d041b368f 100644 --- a/dashboard/components/Relations.tsx +++ b/dashboard/components/Relations.tsx @@ -32,9 +32,9 @@ import Head from "next/head" import Link from "next/link" import { Fragment } from "react" import Title from "../components/Title" +import useFetch from "../lib/api/fetch" +import { Relation, StreamingJob } from "../lib/api/streaming" import extractColumnInfo from "../lib/extractInfo" -import useFetch from "../pages/api/fetch" -import { Relation, StreamingJob } from "../pages/api/streaming" import { Sink as RwSink, Source as RwSource, diff --git a/dashboard/pages/api/api.ts b/dashboard/lib/api/api.ts similarity index 100% rename from dashboard/pages/api/api.ts rename to dashboard/lib/api/api.ts diff --git a/dashboard/pages/api/cluster.ts b/dashboard/lib/api/cluster.ts similarity index 100% rename from dashboard/pages/api/cluster.ts rename to dashboard/lib/api/cluster.ts diff --git a/dashboard/pages/api/fetch.ts b/dashboard/lib/api/fetch.ts similarity index 100% rename from dashboard/pages/api/fetch.ts rename to dashboard/lib/api/fetch.ts diff --git a/dashboard/lib/api/metric.ts b/dashboard/lib/api/metric.ts new file mode 100644 index 0000000000000..d40f92f405df1 --- /dev/null +++ b/dashboard/lib/api/metric.ts @@ -0,0 +1,186 @@ +/* + * Copyright 2024 RisingWave Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ +import { Metrics, MetricsSample } from "../../components/metrics" +import api from "./api" + +export interface BackPressuresMetrics { + outputBufferBlockingDuration: Metrics[] +} + +// Get back pressure from Prometheus +export async function fetchPrometheusBackPressure() { + const res: BackPressuresMetrics = await api.get( + "/metrics/fragment/prometheus_back_pressures" + ) + return res +} + +export interface BackPressureInfo { + actorId: number + fragmentId: number + downstreamFragmentId: number + value: number +} + +export interface BackPressureRateInfo { + actorId: number + fragmentId: number + downstreamFragmentId: number + backPressureRate: number +} + +function convertToMapAndAgg( + backPressures: BackPressureInfo[] +): Map { + // FragmentId-downstreamFragmentId, total value + const mapValue = new Map() + // FragmentId-downstreamFragmentId, total count + const mapNumber = new Map() + // FragmentId-downstreamFragmentId, average value + const map = new Map() + for (const item of backPressures) { + const key = `${item.fragmentId}-${item.downstreamFragmentId}` + if (mapValue.has(key) && mapNumber.has(key)) { + // add || tp avoid NaN and pass check + mapValue.set(key, (mapValue.get(key) || 0) + item.value) + mapNumber.set(key, (mapNumber.get(key) || 0) + 1) + } else { + mapValue.set(key, item.value) + mapNumber.set(key, 1) + } + } + + for (const [key, value] of mapValue) { + map.set(key, value / mapNumber.get(key)!) + } + return map +} + +function convertFromMapAndAgg( + map: Map +): BackPressureRateInfo[] { + const result: BackPressureRateInfo[] = [] + map.forEach((value, key) => { + const [fragmentId, downstreamFragmentId] = key.split("-").map(Number) + const backPressureRateInfo: BackPressureRateInfo = { + actorId: 0, + fragmentId, + downstreamFragmentId, + backPressureRate: value, + } + result.push(backPressureRateInfo) + }) + return result +} + +function convertToBackPressureMetrics( + bpRates: BackPressureRateInfo[] +): BackPressuresMetrics { + const bpMetrics: BackPressuresMetrics = { + outputBufferBlockingDuration: [], + } + for (const item of bpRates) { + bpMetrics.outputBufferBlockingDuration.push({ + metric: { + actorId: item.actorId.toString(), + fragmentId: item.fragmentId.toString(), + downstreamFragmentId: item.downstreamFragmentId.toString(), + }, + sample: [ + { + timestamp: Date.now(), + value: item.backPressureRate, + }, + ], + }) + } + return bpMetrics +} + +export function calculateBPRate( + backPressureNew: BackPressureInfo[], + backPressureOld: BackPressureInfo[], + intervalMs: number +): BackPressuresMetrics { + let mapNew = convertToMapAndAgg(backPressureNew) + let mapOld = convertToMapAndAgg(backPressureOld) + let result = new Map() + mapNew.forEach((value, key) => { + if (mapOld.has(key)) { + result.set( + key, + // The *100 in end of the formular is to convert the BP rate to the value used in web UI drawing + ((value - (mapOld.get(key) || 0)) / + ((intervalMs / 1000) * 1000000000)) * + 100 + ) + } else { + result.set(key, 0) + } + }) + + return convertToBackPressureMetrics(convertFromMapAndAgg(result)) +} + +export const BackPressureInfo = { + fromJSON: (object: any) => { + return { + actorId: isSet(object.actorId) ? Number(object.actorId) : 0, + fragmentId: isSet(object.fragmentId) ? Number(object.fragmentId) : 0, + downstreamFragmentId: isSet(object.downstreamFragmentId) + ? Number(object.downstreamFragmentId) + : 0, + value: isSet(object.value) ? Number(object.value) : 0, + } + }, +} + +// Get back pressure from meta node -> compute node +export async function fetchEmbeddedBackPressure() { + const response = await api.get("/metrics/fragment/embedded_back_pressures") + let backPressureInfos: BackPressureInfo[] = response.backPressureInfos.map( + BackPressureInfo.fromJSON + ) + backPressureInfos = backPressureInfos.sort((a, b) => a.actorId - b.actorId) + return backPressureInfos +} + +function calculatePercentile(samples: MetricsSample[], percentile: number) { + const sorted = samples.sort((a, b) => a.value - b.value) + const index = Math.floor(sorted.length * percentile) + return sorted[index].value +} + +export function p50(samples: MetricsSample[]) { + return calculatePercentile(samples, 0.5) +} + +export function p90(samples: MetricsSample[]) { + return calculatePercentile(samples, 0.9) +} + +export function p95(samples: MetricsSample[]) { + return calculatePercentile(samples, 0.95) +} + +export function p99(samples: MetricsSample[]) { + return calculatePercentile(samples, 0.99) +} + +function isSet(value: any): boolean { + return value !== null && value !== undefined +} diff --git a/dashboard/pages/api/streaming.ts b/dashboard/lib/api/streaming.ts similarity index 100% rename from dashboard/pages/api/streaming.ts rename to dashboard/lib/api/streaming.ts diff --git a/dashboard/lib/layout.ts b/dashboard/lib/layout.ts index 924374341daa8..ca96325e9802f 100644 --- a/dashboard/lib/layout.ts +++ b/dashboard/lib/layout.ts @@ -16,9 +16,9 @@ */ import { max } from "lodash" -import { Relation } from "../pages/api/streaming" import { TableFragments_Fragment } from "../proto/gen/meta" import { GraphNode } from "./algo" +import { Relation } from "./api/streaming" export type Enter = Type extends d3.Selection< any, diff --git a/dashboard/mock-server.js b/dashboard/mock-server.js index 31d67bd87b772..2db52df788e22 100644 --- a/dashboard/mock-server.js +++ b/dashboard/mock-server.js @@ -73,8 +73,8 @@ app.get("/metrics/cluster", (req, res, next) => { res.json(require("./mock/metrics_cluster.json")) }) -app.get("/metrics/actor/back_pressures", (req, res, next) => { - res.json(require("./mock/actor_back_pressures.json")) +app.get("/metrics/fragment/prometheus_back_pressures", (req, res, next) => { + res.json(require("./mock/fragment_prometheus_back_pressures.json")) }) app.get("/monitor/await_tree/1", (req, res, next) => { diff --git a/dashboard/mock/fetch.sh b/dashboard/mock/fetch.sh index 5cc278e01d0c5..ae209fdcf9303 100755 --- a/dashboard/mock/fetch.sh +++ b/dashboard/mock/fetch.sh @@ -17,5 +17,5 @@ curl http://localhost:5691/api/internal_tables > internal_tables.json curl http://localhost:5691/api/sinks > sinks.json curl http://localhost:5691/api/sources > sources.json curl http://localhost:5691/api/metrics/cluster > metrics_cluster.json -curl http://localhost:5691/api/metrics/actor/back_pressures > actor_back_pressures.json +curl http://localhost:5691/api/metrics/fragment/prometheus_back_pressures > fragment_prometheus_back_pressures.json curl http://localhost:5691/api/monitor/await_tree/1 > await_tree_1.json diff --git a/dashboard/next.config.js b/dashboard/next.config.js index ca73de39b634e..54b50aa298faf 100644 --- a/dashboard/next.config.js +++ b/dashboard/next.config.js @@ -19,6 +19,7 @@ * @type {import('next').NextConfig} */ const nextConfig = { + output: "export", trailingSlash: true, } diff --git a/dashboard/package-lock.json b/dashboard/package-lock.json index fad401034f3ba..c512f41a0afea 100644 --- a/dashboard/package-lock.json +++ b/dashboard/package-lock.json @@ -26,7 +26,7 @@ "fabric": "^5.2.1", "framer-motion": "^6.5.1", "lodash": "^4.17.21", - "next": "^13.5.4", + "next": "^14.1.0", "nuqs": "^1.14.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -41,12 +41,12 @@ "@testing-library/jest-dom": "^5.16.2", "@testing-library/react": "^13.3.0", "@types/loadable__component": "^5.13.4", - "@types/node": "^18.7.14", + "@types/node": "20.11.0", "@types/styled-components": "^5.1.26", "@typescript-eslint/eslint-plugin": "^5.52.0", "cors": "^2.8.5", "eslint": "^8.45.0", - "eslint-config-next": "13.4.12", + "eslint-config-next": "^14.1.0", "eslint-config-prettier": "^8.8.0", "eslint-config-standard-with-typescript": "^37.0.0", "eslint-plugin-import": "^2.27.5", @@ -56,7 +56,7 @@ "express": "^4.18.1", "prettier": "^2.7.1", "prettier-plugin-organize-imports": "^3.1.1", - "typescript": "^5.1.6" + "typescript": "^5.3.3" } }, "node_modules/@aashutoshrathi/word-wrap": { @@ -364,24 +364,11 @@ } }, "node_modules/@babel/runtime": { - "version": "7.18.9", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.9.tgz", - "integrity": "sha512-lkqXDcvlFT5rvEjiu6+QYO+1GXrEHRo2LOtS7E4GtX5ESIZOgepqsZBVIj6Pv+a6zqsya9VCgiK1KAK4BvJDAw==", - "dependencies": { - "regenerator-runtime": "^0.13.4" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/runtime-corejs3": { - "version": "7.19.0", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.19.0.tgz", - "integrity": "sha512-JyXXoCu1N8GLuKc2ii8y5RGma5FMpFeO2nAQIe0Yzrbq+rQnN+sFj47auLblR5ka6aHNGPDgv8G/iI2Grb0ldQ==", - "dev": true, + "version": "7.23.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.9.tgz", + "integrity": "sha512-0CX6F+BI2s9dkUqr08KFrAIZgNFj75rdBU/DjCyYLIaV/quFjkk6T+EJ2LkZHyZTbEV4L5p97mNkUsHl2wLFAw==", "dependencies": { - "core-js-pure": "^3.20.2", - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.14.0" }, "engines": { "node": ">=6.9.0" @@ -1741,6 +1728,73 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "node_modules/@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "dependencies": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/@isaacs/cliui/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/@isaacs/cliui/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "node_modules/@isaacs/cliui/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/@isaacs/cliui/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/@jest/expect-utils": { "version": "29.0.2", "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.0.2.tgz", @@ -2039,43 +2093,78 @@ } }, "node_modules/@next/env": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-13.5.4.tgz", - "integrity": "sha512-LGegJkMvRNw90WWphGJ3RMHMVplYcOfRWf2Be3td3sUa+1AaxmsYyANsA+znrGCBjXJNi4XAQlSoEfUxs/4kIQ==" + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.0.tgz", + "integrity": "sha512-Py8zIo+02ht82brwwhTg36iogzFqGLPXlRGKQw5s+qP/kMNc4MAyDeEwBKDijk6zTIbegEgu8Qy7C1LboslQAw==" }, "node_modules/@next/eslint-plugin-next": { - "version": "13.4.12", - "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-13.4.12.tgz", - "integrity": "sha512-6rhK9CdxEgj/j1qvXIyLTWEaeFv7zOK8yJMulz3Owel0uek0U9MJCGzmKgYxM3aAUBo3gKeywCZKyQnJKto60A==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.1.0.tgz", + "integrity": "sha512-x4FavbNEeXx/baD/zC/SdrvkjSby8nBn8KcCREqk6UuwvwoAPZmaV8TFCAuo/cpovBRTIY67mHhe86MQQm/68Q==", + "dev": true, + "dependencies": { + "glob": "10.3.10" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", "dev": true, "dependencies": { - "glob": "7.1.7" + "balanced-match": "^1.0.0" } }, "node_modules/@next/eslint-plugin-next/node_modules/glob": { - "version": "7.1.7", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", - "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "version": "10.3.10", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", + "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", "dev": true, "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.5", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" + }, + "bin": { + "glob": "dist/esm/bin.mjs" }, "engines": { - "node": "*" + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/@next/eslint-plugin-next/node_modules/minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=16 || 14 >=14.17" }, "funding": { "url": "https://github.com/sponsors/isaacs" } }, + "node_modules/@next/eslint-plugin-next/node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "dev": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/@next/swc-darwin-arm64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-13.5.4.tgz", - "integrity": "sha512-Df8SHuXgF1p+aonBMcDPEsaahNo2TCwuie7VXED4FVyECvdXfRT9unapm54NssV9tF3OQFKBFOdlje4T43VO0w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.0.tgz", + "integrity": "sha512-nUDn7TOGcIeyQni6lZHfzNoo9S0euXnu0jhsbMOmMJUBfgsnESdjN97kM7cBqQxZa8L/bM9om/S5/1dzCrW6wQ==", "cpu": [ "arm64" ], @@ -2088,9 +2177,9 @@ } }, "node_modules/@next/swc-darwin-x64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.5.4.tgz", - "integrity": "sha512-siPuUwO45PnNRMeZnSa8n/Lye5ZX93IJom9wQRB5DEOdFrw0JjOMu1GINB8jAEdwa7Vdyn1oJ2xGNaQpdQQ9Pw==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz", + "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==", "cpu": [ "x64" ], @@ -2103,9 +2192,9 @@ } }, "node_modules/@next/swc-linux-arm64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.5.4.tgz", - "integrity": "sha512-l/k/fvRP/zmB2jkFMfefmFkyZbDkYW0mRM/LB+tH5u9pB98WsHXC0WvDHlGCYp3CH/jlkJPL7gN8nkTQVrQ/2w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz", + "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==", "cpu": [ "arm64" ], @@ -2118,9 +2207,9 @@ } }, "node_modules/@next/swc-linux-arm64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.5.4.tgz", - "integrity": "sha512-YYGb7SlLkI+XqfQa8VPErljb7k9nUnhhRrVaOdfJNCaQnHBcvbT7cx/UjDQLdleJcfyg1Hkn5YSSIeVfjgmkTg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz", + "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==", "cpu": [ "arm64" ], @@ -2133,9 +2222,9 @@ } }, "node_modules/@next/swc-linux-x64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.5.4.tgz", - "integrity": "sha512-uE61vyUSClnCH18YHjA8tE1prr/PBFlBFhxBZis4XBRJoR+txAky5d7gGNUIbQ8sZZ7LVkSVgm/5Fc7mwXmRAg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz", + "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==", "cpu": [ "x64" ], @@ -2148,9 +2237,9 @@ } }, "node_modules/@next/swc-linux-x64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.5.4.tgz", - "integrity": "sha512-qVEKFYML/GvJSy9CfYqAdUexA6M5AklYcQCW+8JECmkQHGoPxCf04iMh7CPR7wkHyWWK+XLt4Ja7hhsPJtSnhg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz", + "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==", "cpu": [ "x64" ], @@ -2163,9 +2252,9 @@ } }, "node_modules/@next/swc-win32-arm64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.5.4.tgz", - "integrity": "sha512-mDSQfqxAlfpeZOLPxLymZkX0hYF3juN57W6vFHTvwKlnHfmh12Pt7hPIRLYIShk8uYRsKPtMTth/EzpwRI+u8w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz", + "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==", "cpu": [ "arm64" ], @@ -2178,9 +2267,9 @@ } }, "node_modules/@next/swc-win32-ia32-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.5.4.tgz", - "integrity": "sha512-aoqAT2XIekIWoriwzOmGFAvTtVY5O7JjV21giozBTP5c6uZhpvTWRbmHXbmsjZqY4HnEZQRXWkSAppsIBweKqw==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz", + "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==", "cpu": [ "ia32" ], @@ -2193,9 +2282,9 @@ } }, "node_modules/@next/swc-win32-x64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.5.4.tgz", - "integrity": "sha512-cyRvlAxwlddlqeB9xtPSfNSCRy8BOa4wtMo0IuI9P7Y0XT2qpDrpFKRyZ7kUngZis59mPVla5k8X1oOJ8RxDYg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz", + "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==", "cpu": [ "x64" ], @@ -2242,6 +2331,16 @@ "node": ">= 8" } }, + "node_modules/@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "optional": true, + "engines": { + "node": ">=14" + } + }, "node_modules/@pkgr/utils": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@pkgr/utils/-/utils-2.4.2.tgz", @@ -2326,9 +2425,9 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "node_modules/@rushstack/eslint-patch": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.1.4.tgz", - "integrity": "sha512-LwzQKA4vzIct1zNZzBmRKI9QuNpLgTQMEjsQLf3BXuGYb3QPTP4Yjf6mkdX+X1mYttZ808QpOwAzZjv28kq7DA==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.7.2.tgz", + "integrity": "sha512-RbhOOTCNoCrbfkRyoXODZp75MlpiHMgbE5MEBZAnnnLyQNgrigEj4p0lzsMDyc1zVsJDLrivB58tgg3emX0eEA==", "dev": true }, "node_modules/@sinclair/typebox": { @@ -2896,9 +2995,12 @@ } }, "node_modules/@types/node": { - "version": "18.7.14", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.14.tgz", - "integrity": "sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==" + "version": "20.11.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.0.tgz", + "integrity": "sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==", + "dependencies": { + "undici-types": "~5.26.4" + } }, "node_modules/@types/parse-json": { "version": "4.0.0", @@ -3420,12 +3522,12 @@ } }, "node_modules/aria-query": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.0.2.tgz", - "integrity": "sha512-eigU3vhqSO+Z8BKDnVLN/ompjhf3pYzecKXz8+whRy+9gZu8n1TCGfwzQUUPnqdHl9ax1Hr9031orZ+UOEYr7Q==", + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", "dev": true, - "engines": { - "node": ">=6.0" + "dependencies": { + "dequal": "^2.0.3" } }, "node_modules/array-buffer-byte-length": { @@ -3448,15 +3550,15 @@ "dev": true }, "node_modules/array-includes": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.6.tgz", - "integrity": "sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.7.tgz", + "integrity": "sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4", - "get-intrinsic": "^1.1.3", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", "is-string": "^1.0.7" }, "engines": { @@ -3475,15 +3577,34 @@ "node": ">=8" } }, + "node_modules/array.prototype.findlastindex": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.3.tgz", + "integrity": "sha512-LzLoiOMAxvy+Gd3BAq3B7VeIgPdo+Q8hthvKtXybMvRV0jrXfJM/t8mw7nNlpEcVlVUnCnM2KSX4XU5HmpodOA==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0", + "get-intrinsic": "^1.2.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/array.prototype.flat": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz", - "integrity": "sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz", + "integrity": "sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", "es-shim-unscopables": "^1.0.0" }, "engines": { @@ -3494,14 +3615,14 @@ } }, "node_modules/array.prototype.flatmap": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz", - "integrity": "sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz", + "integrity": "sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", "es-shim-unscopables": "^1.0.0" }, "engines": { @@ -3550,11 +3671,20 @@ "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "node_modules/ast-types-flow": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", - "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.8.tgz", + "integrity": "sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==", "dev": true }, + "node_modules/asynciterator.prototype": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/asynciterator.prototype/-/asynciterator.prototype-1.0.0.tgz", + "integrity": "sha512-wwHYEIS0Q80f5mosx3L/dfG5t5rjEa9Ft51GTaNt862EnpyGHpgz2RkZvLPp1oF5TnAiTohkEKVEu8pQPJI7Vg==", + "dev": true, + "dependencies": { + "has-symbols": "^1.0.3" + } + }, "node_modules/asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -3574,19 +3704,22 @@ } }, "node_modules/axe-core": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz", - "integrity": "sha512-32+ub6kkdhhWick/UjvEwRchgoetXqTK14INLqbGm5U2TzBkBNF3nQtLYm8ovxSkQWArjEQvftCKryjZaATu3w==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz", + "integrity": "sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==", "dev": true, "engines": { "node": ">=4" } }, "node_modules/axobject-query": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz", - "integrity": "sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==", - "dev": true + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.2.1.tgz", + "integrity": "sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==", + "dev": true, + "dependencies": { + "dequal": "^2.0.3" + } }, "node_modules/babel-plugin-macros": { "version": "3.1.0", @@ -3834,13 +3967,14 @@ } }, "node_modules/call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", + "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", "dev": true, "dependencies": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.1", + "set-function-length": "^1.1.1" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -3860,9 +3994,9 @@ "integrity": "sha512-W2lPwkBkMZwFlPCXhIlYgxu+7gC/NUlCtdK652DAJ1JdgV0sTrvuPFshNPrFa1TY2JOkLhgdeEBplB4ezEa+xg==" }, "node_modules/caniuse-lite": { - "version": "1.0.30001517", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001517.tgz", - "integrity": "sha512-Vdhm5S11DaFVLlyiKu4hiUTkpZu+y1KA/rZZqVQfOD5YdDT/eQKlkt7NaE0WGOFgX32diqt9MiP9CAiFeRklaA==", + "version": "1.0.30001581", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001581.tgz", + "integrity": "sha512-whlTkwhqV2tUmP3oYhtNfaWGYHDdS3JYFQBKXxcUR9qqPWsRhFHhoISO2Xnl/g0xyKzht9mI1LZpiNWfMzHixQ==", "funding": [ { "type": "opencollective", @@ -4122,17 +4256,6 @@ "toggle-selection": "^1.0.6" } }, - "node_modules/core-js-pure": { - "version": "3.25.0", - "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.25.0.tgz", - "integrity": "sha512-IeHpLwk3uoci37yoI2Laty59+YqH9x5uR65/yiA0ARAJrTrN4YU0rmauLWfvqOuk77SlNJXj2rM6oT/dBD87+A==", - "dev": true, - "hasInstallScript": true, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/core-js" - } - }, "node_modules/cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", @@ -4742,6 +4865,20 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/define-data-property": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", + "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "dev": true, + "dependencies": { + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/define-lazy-prop": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", @@ -4755,11 +4892,12 @@ } }, "node_modules/define-properties": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.0.tgz", - "integrity": "sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", "dev": true, "dependencies": { + "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" }, @@ -4802,6 +4940,15 @@ "node": ">= 0.8" } }, + "node_modules/dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true, + "engines": { + "node": ">=6" + } + }, "node_modules/destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", @@ -4904,6 +5051,12 @@ "node": ">=0.10" } }, + "node_modules/eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -4920,7 +5073,7 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "optional": true + "devOptional": true }, "node_modules/encodeurl": { "version": "1.0.2", @@ -5005,6 +5158,28 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/es-iterator-helpers": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.0.15.tgz", + "integrity": "sha512-GhoY8uYqd6iwUl2kgjTm4CZAf6oo5mHK7BPqx3rKgx893YSsy0LGHV6gfqqQvZt/8xM8xeOnfXBCfqclMKkJ5g==", + "dev": true, + "dependencies": { + "asynciterator.prototype": "^1.0.0", + "call-bind": "^1.0.2", + "define-properties": "^1.2.1", + "es-abstract": "^1.22.1", + "es-set-tostringtag": "^2.0.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.2.1", + "globalthis": "^1.0.3", + "has-property-descriptors": "^1.0.0", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.5", + "iterator.prototype": "^1.1.2", + "safe-array-concat": "^1.0.1" + } + }, "node_modules/es-set-tostringtag": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz", @@ -5157,20 +5332,20 @@ } }, "node_modules/eslint-config-next": { - "version": "13.4.12", - "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-13.4.12.tgz", - "integrity": "sha512-ZF0r5vxKaVazyZH/37Au/XItiG7qUOBw+HaH3PeyXltIMwXorsn6bdrl0Nn9N5v5v9spc+6GM2ryjugbjF6X2g==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.1.0.tgz", + "integrity": "sha512-SBX2ed7DoRFXC6CQSLc/SbLY9Ut6HxNB2wPTcoIWjUMd7aF7O/SIE7111L8FdZ9TXsNV4pulUDnfthpyPtbFUg==", "dev": true, "dependencies": { - "@next/eslint-plugin-next": "13.4.12", - "@rushstack/eslint-patch": "^1.1.3", - "@typescript-eslint/parser": "^5.42.0", + "@next/eslint-plugin-next": "14.1.0", + "@rushstack/eslint-patch": "^1.3.3", + "@typescript-eslint/parser": "^5.4.2 || ^6.0.0", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-jsx-a11y": "^6.5.1", - "eslint-plugin-react": "^7.31.7", - "eslint-plugin-react-hooks": "5.0.0-canary-7118f5dd7-20230705" + "eslint-plugin-import": "^2.28.1", + "eslint-plugin-jsx-a11y": "^6.7.1", + "eslint-plugin-react": "^7.33.2", + "eslint-plugin-react-hooks": "^4.5.0 || 5.0.0-canary-7118f5dd7-20230705" }, "peerDependencies": { "eslint": "^7.23.0 || ^8.0.0", @@ -5242,14 +5417,14 @@ } }, "node_modules/eslint-import-resolver-node": { - "version": "0.3.7", - "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.7.tgz", - "integrity": "sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==", + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz", + "integrity": "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==", "dev": true, "dependencies": { "debug": "^3.2.7", - "is-core-module": "^2.11.0", - "resolve": "^1.22.1" + "is-core-module": "^2.13.0", + "resolve": "^1.22.4" } }, "node_modules/eslint-import-resolver-node/node_modules/debug": { @@ -5319,9 +5494,9 @@ } }, "node_modules/eslint-module-utils": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.4.tgz", - "integrity": "sha512-j4GT+rqzCoRKHwURX7pddtIPGySnX9Si/cgMI5ztrcqOPtk5dDEeZ34CQVPphnqkJytlc97Vuk05Um2mJ3gEQA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz", + "integrity": "sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==", "dev": true, "dependencies": { "debug": "^3.2.7" @@ -5388,26 +5563,28 @@ } }, "node_modules/eslint-plugin-import": { - "version": "2.27.5", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.27.5.tgz", - "integrity": "sha512-LmEt3GVofgiGuiE+ORpnvP+kAm3h6MLZJ4Q5HCyHADofsb4VzXFsRiWj3c0OFiV+3DWFh0qg3v9gcPlfc3zRow==", + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz", + "integrity": "sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==", "dev": true, "dependencies": { - "array-includes": "^3.1.6", - "array.prototype.flat": "^1.3.1", - "array.prototype.flatmap": "^1.3.1", + "array-includes": "^3.1.7", + "array.prototype.findlastindex": "^1.2.3", + "array.prototype.flat": "^1.3.2", + "array.prototype.flatmap": "^1.3.2", "debug": "^3.2.7", "doctrine": "^2.1.0", - "eslint-import-resolver-node": "^0.3.7", - "eslint-module-utils": "^2.7.4", - "has": "^1.0.3", - "is-core-module": "^2.11.0", + "eslint-import-resolver-node": "^0.3.9", + "eslint-module-utils": "^2.8.0", + "hasown": "^2.0.0", + "is-core-module": "^2.13.1", "is-glob": "^4.0.3", "minimatch": "^3.1.2", - "object.values": "^1.1.6", - "resolve": "^1.22.1", - "semver": "^6.3.0", - "tsconfig-paths": "^3.14.1" + "object.fromentries": "^2.0.7", + "object.groupby": "^1.0.1", + "object.values": "^1.1.7", + "semver": "^6.3.1", + "tsconfig-paths": "^3.15.0" }, "engines": { "node": ">=4" @@ -5438,24 +5615,27 @@ } }, "node_modules/eslint-plugin-jsx-a11y": { - "version": "6.6.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.6.1.tgz", - "integrity": "sha512-sXgFVNHiWffBq23uiS/JaP6eVR622DqwB4yTzKvGZGcPq6/yZ3WmOZfuBks/vHWo9GaFOqC2ZK4i6+C35knx7Q==", - "dev": true, - "dependencies": { - "@babel/runtime": "^7.18.9", - "aria-query": "^4.2.2", - "array-includes": "^3.1.5", - "ast-types-flow": "^0.0.7", - "axe-core": "^4.4.3", - "axobject-query": "^2.2.0", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.8.0.tgz", + "integrity": "sha512-Hdh937BS3KdwwbBaKd5+PLCOmYY6U4f2h9Z2ktwtNKvIdIEu137rjYbcb9ApSbVJfWxANNuiKTD/9tOKjK9qOA==", + "dev": true, + "dependencies": { + "@babel/runtime": "^7.23.2", + "aria-query": "^5.3.0", + "array-includes": "^3.1.7", + "array.prototype.flatmap": "^1.3.2", + "ast-types-flow": "^0.0.8", + "axe-core": "=4.7.0", + "axobject-query": "^3.2.1", "damerau-levenshtein": "^1.0.8", "emoji-regex": "^9.2.2", - "has": "^1.0.3", - "jsx-ast-utils": "^3.3.2", - "language-tags": "^1.0.5", + "es-iterator-helpers": "^1.0.15", + "hasown": "^2.0.0", + "jsx-ast-utils": "^3.3.5", + "language-tags": "^1.0.9", "minimatch": "^3.1.2", - "semver": "^6.3.0" + "object.entries": "^1.1.7", + "object.fromentries": "^2.0.7" }, "engines": { "node": ">=4.0" @@ -5464,19 +5644,6 @@ "eslint": "^3 || ^4 || ^5 || ^6 || ^7 || ^8" } }, - "node_modules/eslint-plugin-jsx-a11y/node_modules/aria-query": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-4.2.2.tgz", - "integrity": "sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA==", - "dev": true, - "dependencies": { - "@babel/runtime": "^7.10.2", - "@babel/runtime-corejs3": "^7.10.2" - }, - "engines": { - "node": ">=6.0" - } - }, "node_modules/eslint-plugin-jsx-a11y/node_modules/emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -5536,15 +5703,16 @@ } }, "node_modules/eslint-plugin-react": { - "version": "7.33.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.33.0.tgz", - "integrity": "sha512-qewL/8P34WkY8jAqdQxsiL82pDUeT7nhs8IsuXgfgnsEloKCT4miAV9N9kGtx7/KM9NH/NCGUE7Edt9iGxLXFw==", + "version": "7.33.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.33.2.tgz", + "integrity": "sha512-73QQMKALArI8/7xGLNI/3LylrEYrlKZSb5C9+q3OtOewTnMQi5cT+aE9E41sLCmli3I9PGGmD1yiZydyo4FEPw==", "dev": true, "dependencies": { "array-includes": "^3.1.6", "array.prototype.flatmap": "^1.3.1", "array.prototype.tosorted": "^1.1.1", "doctrine": "^2.1.0", + "es-iterator-helpers": "^1.0.12", "estraverse": "^5.3.0", "jsx-ast-utils": "^2.4.1 || ^3.0.0", "minimatch": "^3.1.2", @@ -6265,6 +6433,34 @@ "is-callable": "^1.1.3" } }, + "node_modules/foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dev": true, + "dependencies": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/foreground-child/node_modules/signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "node_modules/form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -6375,9 +6571,12 @@ "devOptional": true }, "node_modules/function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/function.prototype.name": { "version": "1.1.5", @@ -6436,15 +6635,15 @@ } }, "node_modules/get-intrinsic": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", - "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", + "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", "dev": true, "dependencies": { - "function-bind": "^1.1.1", - "has": "^1.0.3", + "function-bind": "^1.1.2", "has-proto": "^1.0.1", - "has-symbols": "^1.0.3" + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -6530,11 +6729,6 @@ "node": ">=10.13.0" } }, - "node_modules/glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" - }, "node_modules/globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", @@ -6591,9 +6785,9 @@ } }, "node_modules/graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "node_modules/graphemer": { "version": "1.4.0", @@ -6605,6 +6799,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, "dependencies": { "function-bind": "^1.1.1" }, @@ -6630,12 +6825,12 @@ } }, "node_modules/has-property-descriptors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", - "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", + "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", "dev": true, "dependencies": { - "get-intrinsic": "^1.1.1" + "get-intrinsic": "^1.2.2" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -6686,6 +6881,17 @@ "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", "optional": true }, + "node_modules/hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/hast-util-parse-selector": { "version": "2.2.5", "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", @@ -6950,6 +7156,21 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" }, + "node_modules/is-async-function": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz", + "integrity": "sha512-Y1JXKrfykRJGdlDwdKlLpLyMIiWqWvuSd17TvZk68PLAOGOoF4Xyav1z0Xhoi+gCYjZVeC5SI+hYFOfvXmGRCA==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-bigint": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", @@ -6991,11 +7212,11 @@ } }, "node_modules/is-core-module": { - "version": "2.12.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz", - "integrity": "sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", "dependencies": { - "has": "^1.0.3" + "hasown": "^2.0.0" }, "funding": { "url": "https://github.com/sponsors/ljharb" @@ -7049,15 +7270,42 @@ "node": ">=0.10.0" } }, - "node_modules/is-fullwidth-code-point": { - "version": "3.0.0", + "node_modules/is-finalizationregistry": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.0.2.tgz", + "integrity": "sha512-0by5vtUJs8iFQb5TYUHHPudOR+qXYIMKtiUzvLIZITZUjknFmziyBJuLhVRc+Ds0dREFlskDNJKYIdIzu/9pfw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/is-fullwidth-code-point": { + "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "optional": true, + "devOptional": true, "engines": { "node": ">=8" } }, + "node_modules/is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "dev": true, + "dependencies": { + "has-tostringtag": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-glob": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", @@ -7097,6 +7345,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/is-map": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz", + "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-negative-zero": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", @@ -7164,6 +7421,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-set": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz", + "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-shared-array-buffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", @@ -7233,6 +7499,15 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-weakmap": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", + "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-weakref": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", @@ -7245,6 +7520,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-weakset": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz", + "integrity": "sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -7284,6 +7572,37 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "node_modules/iterator.prototype": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz", + "integrity": "sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w==", + "dev": true, + "dependencies": { + "define-properties": "^1.2.1", + "get-intrinsic": "^1.2.1", + "has-symbols": "^1.0.3", + "reflect.getprototypeof": "^1.0.4", + "set-function-name": "^2.0.1" + } + }, + "node_modules/jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dev": true, + "dependencies": { + "@isaacs/cliui": "^8.0.2" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + }, + "optionalDependencies": { + "@pkgjs/parseargs": "^0.11.0" + } + }, "node_modules/javascript-lp-solver": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/javascript-lp-solver/-/javascript-lp-solver-0.4.24.tgz", @@ -7827,13 +8146,15 @@ } }, "node_modules/jsx-ast-utils": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz", - "integrity": "sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==", + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", + "integrity": "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==", "dev": true, "dependencies": { - "array-includes": "^3.1.5", - "object.assign": "^4.1.3" + "array-includes": "^3.1.6", + "array.prototype.flat": "^1.3.1", + "object.assign": "^4.1.4", + "object.values": "^1.1.6" }, "engines": { "node": ">=4.0" @@ -7846,12 +8167,15 @@ "dev": true }, "node_modules/language-tags": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-1.0.5.tgz", - "integrity": "sha512-qJhlO9cGXi6hBGKoxEG/sKZDAHD5Hnu9Hs4WbOY3pCWXDhw0N8x1NenNzm2EnNLkLkk7J2SdxAkDSbb6ftT+UQ==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-1.0.9.tgz", + "integrity": "sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA==", "dev": true, "dependencies": { - "language-subtag-registry": "~0.3.2" + "language-subtag-registry": "^0.3.20" + }, + "engines": { + "node": ">=0.10" } }, "node_modules/levn": { @@ -8109,10 +8433,13 @@ } }, "node_modules/minimist": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", - "dev": true + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } }, "node_modules/minipass": { "version": "3.3.4", @@ -8212,34 +8539,34 @@ } }, "node_modules/next": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/next/-/next-13.5.4.tgz", - "integrity": "sha512-+93un5S779gho8y9ASQhb/bTkQF17FNQOtXLKAj3lsNgltEcF0C5PMLLncDmH+8X1EnJH1kbqAERa29nRXqhjA==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/next/-/next-14.1.0.tgz", + "integrity": "sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q==", "dependencies": { - "@next/env": "13.5.4", + "@next/env": "14.1.0", "@swc/helpers": "0.5.2", "busboy": "1.6.0", - "caniuse-lite": "^1.0.30001406", + "caniuse-lite": "^1.0.30001579", + "graceful-fs": "^4.2.11", "postcss": "8.4.31", - "styled-jsx": "5.1.1", - "watchpack": "2.4.0" + "styled-jsx": "5.1.1" }, "bin": { "next": "dist/bin/next" }, "engines": { - "node": ">=16.14.0" + "node": ">=18.17.0" }, "optionalDependencies": { - "@next/swc-darwin-arm64": "13.5.4", - "@next/swc-darwin-x64": "13.5.4", - "@next/swc-linux-arm64-gnu": "13.5.4", - "@next/swc-linux-arm64-musl": "13.5.4", - "@next/swc-linux-x64-gnu": "13.5.4", - "@next/swc-linux-x64-musl": "13.5.4", - "@next/swc-win32-arm64-msvc": "13.5.4", - "@next/swc-win32-ia32-msvc": "13.5.4", - "@next/swc-win32-x64-msvc": "13.5.4" + "@next/swc-darwin-arm64": "14.1.0", + "@next/swc-darwin-x64": "14.1.0", + "@next/swc-linux-arm64-gnu": "14.1.0", + "@next/swc-linux-arm64-musl": "14.1.0", + "@next/swc-linux-x64-gnu": "14.1.0", + "@next/swc-linux-x64-musl": "14.1.0", + "@next/swc-win32-arm64-msvc": "14.1.0", + "@next/swc-win32-ia32-msvc": "14.1.0", + "@next/swc-win32-x64-msvc": "14.1.0" }, "peerDependencies": { "@opentelemetry/api": "^1.1.0", @@ -8416,28 +8743,28 @@ } }, "node_modules/object.entries": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.6.tgz", - "integrity": "sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w==", + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.7.tgz", + "integrity": "sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" }, "engines": { "node": ">= 0.4" } }, "node_modules/object.fromentries": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.6.tgz", - "integrity": "sha512-VciD13dswC4j1Xt5394WR4MzmAQmlgN72phd/riNp9vtD7tp4QQWJ0R4wvclXcafgcYK8veHRed2W6XeGBvcfg==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.7.tgz", + "integrity": "sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" }, "engines": { "node": ">= 0.4" @@ -8446,6 +8773,18 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/object.groupby": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object.groupby/-/object.groupby-1.0.1.tgz", + "integrity": "sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1" + } + }, "node_modules/object.hasown": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/object.hasown/-/object.hasown-1.1.2.tgz", @@ -8460,14 +8799,14 @@ } }, "node_modules/object.values": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.6.tgz", - "integrity": "sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==", + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.7.tgz", + "integrity": "sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==", "dev": true, "dependencies": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" }, "engines": { "node": ">= 0.4" @@ -8669,6 +9008,40 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, + "node_modules/path-scurry": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz", + "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==", + "dev": true, + "dependencies": { + "lru-cache": "^9.1.1 || ^10.0.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "engines": { + "node": ">=16 || 14 >=14.17" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/path-scurry/node_modules/lru-cache": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "dev": true, + "engines": { + "node": "14 || >=16.14" + } + }, + "node_modules/path-scurry/node_modules/minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "dev": true, + "engines": { + "node": ">=16 || 14 >=14.17" + } + }, "node_modules/path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", @@ -9342,6 +9715,26 @@ "resolved": "https://registry.npmjs.org/postcss-value-parser/-/postcss-value-parser-3.3.1.tgz", "integrity": "sha512-pISE66AbVkp4fDQ7VHBwRNXzAAKJjw4Vw7nWI/+Q3vuly7SNfgYXvm6i5IgFylHGK5sP/xHAbB7N49OS4gWNyQ==" }, + "node_modules/reflect.getprototypeof": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz", + "integrity": "sha512-ECkTw8TmJwW60lOTR+ZkODISW6RQ8+2CL3COqtiJKLd6MmB45hN51HprHFziKLGkAuTGQhBb91V8cy+KHlaCjw==", + "dev": true, + "dependencies": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "globalthis": "^1.0.3", + "which-builtin-type": "^1.1.3" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/refractor": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", @@ -9365,9 +9758,9 @@ } }, "node_modules/regenerator-runtime": { - "version": "0.13.9", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", - "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, "node_modules/regexp.prototype.flags": { "version": "1.5.0", @@ -9405,11 +9798,11 @@ "optional": true }, "node_modules/resolve": { - "version": "1.22.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", - "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", "dependencies": { - "is-core-module": "^2.9.0", + "is-core-module": "^2.13.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" }, @@ -9600,13 +9993,13 @@ "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==" }, "node_modules/safe-array-concat": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.0.0.tgz", - "integrity": "sha512-9dVEFruWIsnie89yym+xWTAYASdpw3CJV7Li/6zBewGf9z2i1j31rP6jnY0pHEO4QZh6N0K11bFjWmdR8UGdPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.0.tgz", + "integrity": "sha512-ZdQ0Jeb9Ofti4hbt5lX3T2JcAamT9hfzYU1MNB+z/jaEbB6wfFfPIR/zEORmZqobkCCJhSjodobH6WHNmJ97dg==", "dev": true, "dependencies": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.2.0", + "call-bind": "^1.0.5", + "get-intrinsic": "^1.2.2", "has-symbols": "^1.0.3", "isarray": "^2.0.5" }, @@ -9735,6 +10128,36 @@ "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", "optional": true }, + "node_modules/set-function-length": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", + "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", + "dev": true, + "dependencies": { + "define-data-property": "^1.1.1", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.2", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/set-function-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", + "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", + "dev": true, + "dependencies": { + "define-data-property": "^1.0.1", + "functions-have-names": "^1.2.3", + "has-property-descriptors": "^1.0.0" + }, + "engines": { + "node": ">= 0.4" + } + }, "node_modules/setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", @@ -9933,7 +10356,22 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "optional": true, + "devOptional": true, + "dependencies": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + }, + "engines": { + "node": ">=8" + } + }, + "node_modules/string-width-cjs": { + "name": "string-width", + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "dependencies": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -10019,6 +10457,19 @@ "node": ">=8" } }, + "node_modules/strip-ansi-cjs": { + "name": "strip-ansi", + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "dependencies": { + "ansi-regex": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", @@ -10333,13 +10784,13 @@ } }, "node_modules/tsconfig-paths": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz", - "integrity": "sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==", + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", + "integrity": "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==", "dev": true, "dependencies": { "@types/json5": "^0.0.29", - "json5": "^1.0.1", + "json5": "^1.0.2", "minimist": "^1.2.6", "strip-bom": "^3.0.0" } @@ -10485,9 +10936,9 @@ } }, "node_modules/typescript": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", - "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", "dev": true, "bin": { "tsc": "bin/tsc", @@ -10530,6 +10981,11 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "node_modules/universalify": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", @@ -10746,18 +11202,6 @@ "node": ">=12" } }, - "node_modules/watchpack": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", - "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", - "dependencies": { - "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.1.2" - }, - "engines": { - "node": ">=10.13.0" - } - }, "node_modules/webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -10832,6 +11276,47 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/which-builtin-type": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.1.3.tgz", + "integrity": "sha512-YmjsSMDBYsM1CaFiayOVT06+KJeXf0o5M/CAd4o1lTadFAtacTUM49zoYxr/oroopFDfhvN6iEcBxUyc3gvKmw==", + "dev": true, + "dependencies": { + "function.prototype.name": "^1.1.5", + "has-tostringtag": "^1.0.0", + "is-async-function": "^2.0.0", + "is-date-object": "^1.0.5", + "is-finalizationregistry": "^1.0.2", + "is-generator-function": "^1.0.10", + "is-regex": "^1.1.4", + "is-weakref": "^1.0.2", + "isarray": "^2.0.5", + "which-boxed-primitive": "^1.0.2", + "which-collection": "^1.0.1", + "which-typed-array": "^1.1.9" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/which-collection": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", + "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", + "dev": true, + "dependencies": { + "is-map": "^2.0.1", + "is-set": "^2.0.1", + "is-weakmap": "^2.0.1", + "is-weakset": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/which-typed-array": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", @@ -10869,6 +11354,136 @@ "node": ">=0.10.0" } }, + "node_modules/wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "dependencies": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs": { + "name": "wrap-ansi", + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "dependencies": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/chalk/wrap-ansi?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "dependencies": { + "color-convert": "^2.0.1" + }, + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/wrap-ansi-cjs/node_modules/color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + }, + "node_modules/wrap-ansi/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-styles?sponsor=1" + } + }, + "node_modules/wrap-ansi/node_modules/emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "node_modules/wrap-ansi/node_modules/string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/wrap-ansi/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", @@ -11185,21 +11800,11 @@ } }, "@babel/runtime": { - "version": "7.18.9", - "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.18.9.tgz", - "integrity": "sha512-lkqXDcvlFT5rvEjiu6+QYO+1GXrEHRo2LOtS7E4GtX5ESIZOgepqsZBVIj6Pv+a6zqsya9VCgiK1KAK4BvJDAw==", + "version": "7.23.9", + "resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.23.9.tgz", + "integrity": "sha512-0CX6F+BI2s9dkUqr08KFrAIZgNFj75rdBU/DjCyYLIaV/quFjkk6T+EJ2LkZHyZTbEV4L5p97mNkUsHl2wLFAw==", "requires": { - "regenerator-runtime": "^0.13.4" - } - }, - "@babel/runtime-corejs3": { - "version": "7.19.0", - "resolved": "https://registry.npmjs.org/@babel/runtime-corejs3/-/runtime-corejs3-7.19.0.tgz", - "integrity": "sha512-JyXXoCu1N8GLuKc2ii8y5RGma5FMpFeO2nAQIe0Yzrbq+rQnN+sFj47auLblR5ka6aHNGPDgv8G/iI2Grb0ldQ==", - "dev": true, - "requires": { - "core-js-pure": "^3.20.2", - "regenerator-runtime": "^0.13.4" + "regenerator-runtime": "^0.14.0" } }, "@babel/template": { @@ -12225,6 +12830,54 @@ "integrity": "sha512-ZnQMnLV4e7hDlUvw8H+U8ASL02SS2Gn6+9Ac3wGGLIe7+je2AeAOxPY+izIPJDfFDb7eDjev0Us8MO1iFRN8hA==", "dev": true }, + "@isaacs/cliui": { + "version": "8.0.2", + "resolved": "https://registry.npmjs.org/@isaacs/cliui/-/cliui-8.0.2.tgz", + "integrity": "sha512-O8jcjabXaleOG9DQ0+ARXWZBTfnP4WNAqzuiJK7ll44AmxGKv/J2M4TPjxjY3znBCfvBXFzucm1twdyFybFqEA==", + "dev": true, + "requires": { + "string-width": "^5.1.2", + "string-width-cjs": "npm:string-width@^4.2.0", + "strip-ansi": "^7.0.1", + "strip-ansi-cjs": "npm:strip-ansi@^6.0.1", + "wrap-ansi": "^8.1.0", + "wrap-ansi-cjs": "npm:wrap-ansi@^7.0.0" + }, + "dependencies": { + "ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "requires": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + } + }, + "strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "requires": { + "ansi-regex": "^6.0.1" + } + } + } + }, "@jest/expect-utils": { "version": "29.0.2", "resolved": "https://registry.npmjs.org/@jest/expect-utils/-/expect-utils-29.0.2.tgz", @@ -12463,87 +13116,110 @@ } }, "@next/env": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/env/-/env-13.5.4.tgz", - "integrity": "sha512-LGegJkMvRNw90WWphGJ3RMHMVplYcOfRWf2Be3td3sUa+1AaxmsYyANsA+znrGCBjXJNi4XAQlSoEfUxs/4kIQ==" + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/env/-/env-14.1.0.tgz", + "integrity": "sha512-Py8zIo+02ht82brwwhTg36iogzFqGLPXlRGKQw5s+qP/kMNc4MAyDeEwBKDijk6zTIbegEgu8Qy7C1LboslQAw==" }, "@next/eslint-plugin-next": { - "version": "13.4.12", - "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-13.4.12.tgz", - "integrity": "sha512-6rhK9CdxEgj/j1qvXIyLTWEaeFv7zOK8yJMulz3Owel0uek0U9MJCGzmKgYxM3aAUBo3gKeywCZKyQnJKto60A==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/eslint-plugin-next/-/eslint-plugin-next-14.1.0.tgz", + "integrity": "sha512-x4FavbNEeXx/baD/zC/SdrvkjSby8nBn8KcCREqk6UuwvwoAPZmaV8TFCAuo/cpovBRTIY67mHhe86MQQm/68Q==", "dev": true, "requires": { - "glob": "7.1.7" + "glob": "10.3.10" }, "dependencies": { + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + }, "glob": { - "version": "7.1.7", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", - "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", + "version": "10.3.10", + "resolved": "https://registry.npmjs.org/glob/-/glob-10.3.10.tgz", + "integrity": "sha512-fa46+tv1Ak0UPK1TOy/pZrIybNNt4HCv7SDzwyfiOZkvZLEbjsZkJBPtDHVshZjbecAoAGSC20MjLDG/qr679g==", "dev": true, "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" + "foreground-child": "^3.1.0", + "jackspeak": "^2.3.5", + "minimatch": "^9.0.1", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0", + "path-scurry": "^1.10.1" } + }, + "minimatch": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-9.0.3.tgz", + "integrity": "sha512-RHiac9mvaRw0x3AYRgDC1CxAP7HTcNrrECeA8YYJeWnpo+2Q5CegtZjaotWTWxDG3UeGA1coE05iH1mPjT/2mg==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + } + }, + "minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "dev": true } } }, "@next/swc-darwin-arm64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-13.5.4.tgz", - "integrity": "sha512-Df8SHuXgF1p+aonBMcDPEsaahNo2TCwuie7VXED4FVyECvdXfRT9unapm54NssV9tF3OQFKBFOdlje4T43VO0w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.1.0.tgz", + "integrity": "sha512-nUDn7TOGcIeyQni6lZHfzNoo9S0euXnu0jhsbMOmMJUBfgsnESdjN97kM7cBqQxZa8L/bM9om/S5/1dzCrW6wQ==", "optional": true }, "@next/swc-darwin-x64": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-13.5.4.tgz", - "integrity": "sha512-siPuUwO45PnNRMeZnSa8n/Lye5ZX93IJom9wQRB5DEOdFrw0JjOMu1GINB8jAEdwa7Vdyn1oJ2xGNaQpdQQ9Pw==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.1.0.tgz", + "integrity": "sha512-1jgudN5haWxiAl3O1ljUS2GfupPmcftu2RYJqZiMJmmbBT5M1XDffjUtRUzP4W3cBHsrvkfOFdQ71hAreNQP6g==", "optional": true }, "@next/swc-linux-arm64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-13.5.4.tgz", - "integrity": "sha512-l/k/fvRP/zmB2jkFMfefmFkyZbDkYW0mRM/LB+tH5u9pB98WsHXC0WvDHlGCYp3CH/jlkJPL7gN8nkTQVrQ/2w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.1.0.tgz", + "integrity": "sha512-RHo7Tcj+jllXUbK7xk2NyIDod3YcCPDZxj1WLIYxd709BQ7WuRYl3OWUNG+WUfqeQBds6kvZYlc42NJJTNi4tQ==", "optional": true }, "@next/swc-linux-arm64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-13.5.4.tgz", - "integrity": "sha512-YYGb7SlLkI+XqfQa8VPErljb7k9nUnhhRrVaOdfJNCaQnHBcvbT7cx/UjDQLdleJcfyg1Hkn5YSSIeVfjgmkTg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.1.0.tgz", + "integrity": "sha512-v6kP8sHYxjO8RwHmWMJSq7VZP2nYCkRVQ0qolh2l6xroe9QjbgV8siTbduED4u0hlk0+tjS6/Tuy4n5XCp+l6g==", "optional": true }, "@next/swc-linux-x64-gnu": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-13.5.4.tgz", - "integrity": "sha512-uE61vyUSClnCH18YHjA8tE1prr/PBFlBFhxBZis4XBRJoR+txAky5d7gGNUIbQ8sZZ7LVkSVgm/5Fc7mwXmRAg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.1.0.tgz", + "integrity": "sha512-zJ2pnoFYB1F4vmEVlb/eSe+VH679zT1VdXlZKX+pE66grOgjmKJHKacf82g/sWE4MQ4Rk2FMBCRnX+l6/TVYzQ==", "optional": true }, "@next/swc-linux-x64-musl": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-13.5.4.tgz", - "integrity": "sha512-qVEKFYML/GvJSy9CfYqAdUexA6M5AklYcQCW+8JECmkQHGoPxCf04iMh7CPR7wkHyWWK+XLt4Ja7hhsPJtSnhg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.1.0.tgz", + "integrity": "sha512-rbaIYFt2X9YZBSbH/CwGAjbBG2/MrACCVu2X0+kSykHzHnYH5FjHxwXLkcoJ10cX0aWCEynpu+rP76x0914atg==", "optional": true }, "@next/swc-win32-arm64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-13.5.4.tgz", - "integrity": "sha512-mDSQfqxAlfpeZOLPxLymZkX0hYF3juN57W6vFHTvwKlnHfmh12Pt7hPIRLYIShk8uYRsKPtMTth/EzpwRI+u8w==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.1.0.tgz", + "integrity": "sha512-o1N5TsYc8f/HpGt39OUQpQ9AKIGApd3QLueu7hXk//2xq5Z9OxmV6sQfNp8C7qYmiOlHYODOGqNNa0e9jvchGQ==", "optional": true }, "@next/swc-win32-ia32-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-13.5.4.tgz", - "integrity": "sha512-aoqAT2XIekIWoriwzOmGFAvTtVY5O7JjV21giozBTP5c6uZhpvTWRbmHXbmsjZqY4HnEZQRXWkSAppsIBweKqw==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.1.0.tgz", + "integrity": "sha512-XXIuB1DBRCFwNO6EEzCTMHT5pauwaSj4SWs7CYnME57eaReAKBXCnkUE80p/pAZcewm7hs+vGvNqDPacEXHVkw==", "optional": true }, "@next/swc-win32-x64-msvc": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-13.5.4.tgz", - "integrity": "sha512-cyRvlAxwlddlqeB9xtPSfNSCRy8BOa4wtMo0IuI9P7Y0XT2qpDrpFKRyZ7kUngZis59mPVla5k8X1oOJ8RxDYg==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.1.0.tgz", + "integrity": "sha512-9WEbVRRAqJ3YFVqEZIxUqkiO8l1nool1LmNxygr5HWF8AcSYsEpneUDhmjUVJEzO2A04+oPtZdombzzPPkTtgg==", "optional": true }, "@nodelib/fs.scandir": { @@ -12572,6 +13248,13 @@ "fastq": "^1.6.0" } }, + "@pkgjs/parseargs": { + "version": "0.11.0", + "resolved": "https://registry.npmjs.org/@pkgjs/parseargs/-/parseargs-0.11.0.tgz", + "integrity": "sha512-+1VkjdD0QBLPodGrJUeqarH8VAIvQODIbwh9XpP5Syisf7YoQgsJKPNFoqqLQlu+VQ/tVSshMR6loPMn8U+dPg==", + "dev": true, + "optional": true + }, "@pkgr/utils": { "version": "2.4.2", "resolved": "https://registry.npmjs.org/@pkgr/utils/-/utils-2.4.2.tgz", @@ -12646,9 +13329,9 @@ "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==" }, "@rushstack/eslint-patch": { - "version": "1.1.4", - "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.1.4.tgz", - "integrity": "sha512-LwzQKA4vzIct1zNZzBmRKI9QuNpLgTQMEjsQLf3BXuGYb3QPTP4Yjf6mkdX+X1mYttZ808QpOwAzZjv28kq7DA==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/@rushstack/eslint-patch/-/eslint-patch-1.7.2.tgz", + "integrity": "sha512-RbhOOTCNoCrbfkRyoXODZp75MlpiHMgbE5MEBZAnnnLyQNgrigEj4p0lzsMDyc1zVsJDLrivB58tgg3emX0eEA==", "dev": true }, "@sinclair/typebox": { @@ -13156,9 +13839,12 @@ } }, "@types/node": { - "version": "18.7.14", - "resolved": "https://registry.npmjs.org/@types/node/-/node-18.7.14.tgz", - "integrity": "sha512-6bbDaETVi8oyIARulOE9qF1/Qdi/23z6emrUh0fNJRUmjznqrixD4MpGDdgOFk5Xb0m2H6Xu42JGdvAxaJR/wA==" + "version": "20.11.0", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.0.tgz", + "integrity": "sha512-o9bjXmDNcF7GbM4CNQpmi+TutCgap/K3w1JyKgxAjqx41zp9qlIAVFi0IhCNsJcXolEqLWhbFbEeL0PvYm4pcQ==", + "requires": { + "undici-types": "~5.26.4" + } }, "@types/parse-json": { "version": "4.0.0", @@ -13527,10 +14213,13 @@ } }, "aria-query": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.0.2.tgz", - "integrity": "sha512-eigU3vhqSO+Z8BKDnVLN/ompjhf3pYzecKXz8+whRy+9gZu8n1TCGfwzQUUPnqdHl9ax1Hr9031orZ+UOEYr7Q==", - "dev": true + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-5.3.0.tgz", + "integrity": "sha512-b0P0sZPKtyu8HkeRAfCq0IfURZK+SuwMjY1UXGBU27wpAiTwQAIlq56IbIO+ytk/JjS1fMR14ee5WBBfKi5J6A==", + "dev": true, + "requires": { + "dequal": "^2.0.3" + } }, "array-buffer-byte-length": { "version": "1.0.0", @@ -13549,15 +14238,15 @@ "dev": true }, "array-includes": { - "version": "3.1.6", - "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.6.tgz", - "integrity": "sha512-sgTbLvL6cNnw24FnbaDyjmvddQ2ML8arZsgaJhoABMoplz/4QRhtrYS+alr1BUM1Bwp6dhx8vVCBSLG+StwOFw==", + "version": "3.1.7", + "resolved": "https://registry.npmjs.org/array-includes/-/array-includes-3.1.7.tgz", + "integrity": "sha512-dlcsNBIiWhPkHdOEEKnehA+RNUWDc4UqFtnIXU4uuYDPtA4LDkr7qip2p0VvFAEXNDr0yWZ9PJyIRiGjRLQzwQ==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4", - "get-intrinsic": "^1.1.3", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", "is-string": "^1.0.7" } }, @@ -13567,27 +14256,40 @@ "integrity": "sha512-HGyxoOTYUyCM6stUe6EJgnd4EoewAI7zMdfqO+kGjnlZmBDz/cR5pf8r/cR4Wq60sL/p0IkcjUEEPwS3GFrIyw==", "dev": true }, + "array.prototype.findlastindex": { + "version": "1.2.3", + "resolved": "https://registry.npmjs.org/array.prototype.findlastindex/-/array.prototype.findlastindex-1.2.3.tgz", + "integrity": "sha512-LzLoiOMAxvy+Gd3BAq3B7VeIgPdo+Q8hthvKtXybMvRV0jrXfJM/t8mw7nNlpEcVlVUnCnM2KSX4XU5HmpodOA==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "es-shim-unscopables": "^1.0.0", + "get-intrinsic": "^1.2.1" + } + }, "array.prototype.flat": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.1.tgz", - "integrity": "sha512-roTU0KWIOmJ4DRLmwKd19Otg0/mT3qPNt0Qb3GWW8iObuZXxrjB/pzn0R3hqpRSWg4HCwqx+0vwOnWnvlOyeIA==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flat/-/array.prototype.flat-1.3.2.tgz", + "integrity": "sha512-djYB+Zx2vLewY8RWlNCUdHjDXs2XOgm602S9E7P/UpHgfeHL00cRiIF+IN/G/aUJ7kGPb6yO/ErDI5V2s8iycA==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", "es-shim-unscopables": "^1.0.0" } }, "array.prototype.flatmap": { - "version": "1.3.1", - "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.1.tgz", - "integrity": "sha512-8UGn9O1FDVvMNB0UlLv4voxRMze7+FpHyF5mSMRjWHUMlpoDViniy05870VlxhfgTnLbpuwTzvD76MTtWxB/mQ==", + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/array.prototype.flatmap/-/array.prototype.flatmap-1.3.2.tgz", + "integrity": "sha512-Ewyx0c9PmpcsByhSW4r+9zDU7sGjFc86qf/kKtuSCRdhfbk0SNLLkaT5qvcHnRGgc5NP/ly/y+qkXkqONX54CQ==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", "es-shim-unscopables": "^1.0.0" } }, @@ -13624,11 +14326,20 @@ "integrity": "sha512-BSHWgDSAiKs50o2Re8ppvp3seVHXSRM44cdSsT9FfNEUUZLOGWVCsiWaRPWM1Znn+mqZ1OfVZ3z3DWEzSp7hRA==" }, "ast-types-flow": { - "version": "0.0.7", - "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.7.tgz", - "integrity": "sha512-eBvWn1lvIApYMhzQMsu9ciLfkBY499mFZlNqG+/9WR7PVlroQw0vG30cOQQbaKz3sCEc44TAOu2ykzqXSNnwag==", + "version": "0.0.8", + "resolved": "https://registry.npmjs.org/ast-types-flow/-/ast-types-flow-0.0.8.tgz", + "integrity": "sha512-OH/2E5Fg20h2aPrbe+QL8JZQFko0YZaF+j4mnQ7BGhfavO7OpSLa8a0y9sBwomHdSbkhTS8TQNayBfnW5DwbvQ==", "dev": true }, + "asynciterator.prototype": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/asynciterator.prototype/-/asynciterator.prototype-1.0.0.tgz", + "integrity": "sha512-wwHYEIS0Q80f5mosx3L/dfG5t5rjEa9Ft51GTaNt862EnpyGHpgz2RkZvLPp1oF5TnAiTohkEKVEu8pQPJI7Vg==", + "dev": true, + "requires": { + "has-symbols": "^1.0.3" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -13642,16 +14353,19 @@ "dev": true }, "axe-core": { - "version": "4.4.3", - "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.4.3.tgz", - "integrity": "sha512-32+ub6kkdhhWick/UjvEwRchgoetXqTK14INLqbGm5U2TzBkBNF3nQtLYm8ovxSkQWArjEQvftCKryjZaATu3w==", + "version": "4.7.0", + "resolved": "https://registry.npmjs.org/axe-core/-/axe-core-4.7.0.tgz", + "integrity": "sha512-M0JtH+hlOL5pLQwHOLNYZaXuhqmvS8oExsqB1SBYgA4Dk7u/xx+YdGHXaK5pyUfed5mYXdlYiphWq3G8cRi5JQ==", "dev": true }, "axobject-query": { - "version": "2.2.0", - "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-2.2.0.tgz", - "integrity": "sha512-Td525n+iPOOyUQIeBfcASuG6uJsDOITl7Mds5gFyerkWiX7qhUTdYUBlSgNMyVqtSJqwpt1kXGLdUt6SykLMRA==", - "dev": true + "version": "3.2.1", + "resolved": "https://registry.npmjs.org/axobject-query/-/axobject-query-3.2.1.tgz", + "integrity": "sha512-jsyHu61e6N4Vbz/v18DHwWYKK0bSWLqn47eeDSKPB7m8tqMHF9YJ+mhIk2lVteyZrY8tnSj/jHOv4YiTCuCJgg==", + "dev": true, + "requires": { + "dequal": "^2.0.3" + } }, "babel-plugin-macros": { "version": "3.1.0", @@ -13843,13 +14557,14 @@ "dev": true }, "call-bind": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", - "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.5.tgz", + "integrity": "sha512-C3nQxfFZxFRVoJoGKKI8y3MOEo129NQ+FgQ08iye+Mk4zNZZGdjfs06bVTr+DBSlA66Q2VEcMki/cUCP4SercQ==", "dev": true, "requires": { - "function-bind": "^1.1.1", - "get-intrinsic": "^1.0.2" + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.1", + "set-function-length": "^1.1.1" } }, "callsites": { @@ -13863,9 +14578,9 @@ "integrity": "sha512-W2lPwkBkMZwFlPCXhIlYgxu+7gC/NUlCtdK652DAJ1JdgV0sTrvuPFshNPrFa1TY2JOkLhgdeEBplB4ezEa+xg==" }, "caniuse-lite": { - "version": "1.0.30001517", - "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001517.tgz", - "integrity": "sha512-Vdhm5S11DaFVLlyiKu4hiUTkpZu+y1KA/rZZqVQfOD5YdDT/eQKlkt7NaE0WGOFgX32diqt9MiP9CAiFeRklaA==" + "version": "1.0.30001581", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001581.tgz", + "integrity": "sha512-whlTkwhqV2tUmP3oYhtNfaWGYHDdS3JYFQBKXxcUR9qqPWsRhFHhoISO2Xnl/g0xyKzht9mI1LZpiNWfMzHixQ==" }, "canvas": { "version": "2.9.3", @@ -14048,12 +14763,6 @@ "toggle-selection": "^1.0.6" } }, - "core-js-pure": { - "version": "3.25.0", - "resolved": "https://registry.npmjs.org/core-js-pure/-/core-js-pure-3.25.0.tgz", - "integrity": "sha512-IeHpLwk3uoci37yoI2Laty59+YqH9x5uR65/yiA0ARAJrTrN4YU0rmauLWfvqOuk77SlNJXj2rM6oT/dBD87+A==", - "dev": true - }, "cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", @@ -14516,6 +15225,17 @@ "untildify": "^4.0.0" } }, + "define-data-property": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.1.tgz", + "integrity": "sha512-E7uGkTzkk1d0ByLeSc6ZsFS79Axg+m1P/VsgYsxHgiuc3tFSj+MjMIwe90FC4lOAZzNBdY7kkO2P2wKdsQ1vgQ==", + "dev": true, + "requires": { + "get-intrinsic": "^1.2.1", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.0" + } + }, "define-lazy-prop": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/define-lazy-prop/-/define-lazy-prop-3.0.0.tgz", @@ -14523,11 +15243,12 @@ "dev": true }, "define-properties": { - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.0.tgz", - "integrity": "sha512-xvqAVKGfT1+UAvPwKTVw/njhdQ8ZhXK4lI0bCIuCMrp2up9nPnaDftrLtmpTazqd1o+UY4zgzU+avtMbDP+ldA==", + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", "dev": true, "requires": { + "define-data-property": "^1.0.1", "has-property-descriptors": "^1.0.0", "object-keys": "^1.1.1" } @@ -14558,6 +15279,12 @@ "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==", "dev": true }, + "dequal": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/dequal/-/dequal-2.0.3.tgz", + "integrity": "sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==", + "dev": true + }, "destroy": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", @@ -14637,6 +15364,12 @@ } } }, + "eastasianwidth": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", + "integrity": "sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==", + "dev": true + }, "ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -14653,7 +15386,7 @@ "version": "8.0.0", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-8.0.0.tgz", "integrity": "sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A==", - "optional": true + "devOptional": true }, "encodeurl": { "version": "1.0.2", @@ -14726,6 +15459,28 @@ "which-typed-array": "^1.1.10" } }, + "es-iterator-helpers": { + "version": "1.0.15", + "resolved": "https://registry.npmjs.org/es-iterator-helpers/-/es-iterator-helpers-1.0.15.tgz", + "integrity": "sha512-GhoY8uYqd6iwUl2kgjTm4CZAf6oo5mHK7BPqx3rKgx893YSsy0LGHV6gfqqQvZt/8xM8xeOnfXBCfqclMKkJ5g==", + "dev": true, + "requires": { + "asynciterator.prototype": "^1.0.0", + "call-bind": "^1.0.2", + "define-properties": "^1.2.1", + "es-abstract": "^1.22.1", + "es-set-tostringtag": "^2.0.1", + "function-bind": "^1.1.1", + "get-intrinsic": "^1.2.1", + "globalthis": "^1.0.3", + "has-property-descriptors": "^1.0.0", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "internal-slot": "^1.0.5", + "iterator.prototype": "^1.1.2", + "safe-array-concat": "^1.0.1" + } + }, "es-set-tostringtag": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.0.1.tgz", @@ -14950,20 +15705,20 @@ } }, "eslint-config-next": { - "version": "13.4.12", - "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-13.4.12.tgz", - "integrity": "sha512-ZF0r5vxKaVazyZH/37Au/XItiG7qUOBw+HaH3PeyXltIMwXorsn6bdrl0Nn9N5v5v9spc+6GM2ryjugbjF6X2g==", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-14.1.0.tgz", + "integrity": "sha512-SBX2ed7DoRFXC6CQSLc/SbLY9Ut6HxNB2wPTcoIWjUMd7aF7O/SIE7111L8FdZ9TXsNV4pulUDnfthpyPtbFUg==", "dev": true, "requires": { - "@next/eslint-plugin-next": "13.4.12", - "@rushstack/eslint-patch": "^1.1.3", - "@typescript-eslint/parser": "^5.42.0", + "@next/eslint-plugin-next": "14.1.0", + "@rushstack/eslint-patch": "^1.3.3", + "@typescript-eslint/parser": "^5.4.2 || ^6.0.0", "eslint-import-resolver-node": "^0.3.6", "eslint-import-resolver-typescript": "^3.5.2", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-jsx-a11y": "^6.5.1", - "eslint-plugin-react": "^7.31.7", - "eslint-plugin-react-hooks": "5.0.0-canary-7118f5dd7-20230705" + "eslint-plugin-import": "^2.28.1", + "eslint-plugin-jsx-a11y": "^6.7.1", + "eslint-plugin-react": "^7.33.2", + "eslint-plugin-react-hooks": "^4.5.0 || 5.0.0-canary-7118f5dd7-20230705" } }, "eslint-config-prettier": { @@ -14991,14 +15746,14 @@ } }, "eslint-import-resolver-node": { - "version": "0.3.7", - "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.7.tgz", - "integrity": "sha512-gozW2blMLJCeFpBwugLTGyvVjNoeo1knonXAcatC6bjPBZitotxdWf7Gimr25N4c0AAOo4eOUfaG82IJPDpqCA==", + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/eslint-import-resolver-node/-/eslint-import-resolver-node-0.3.9.tgz", + "integrity": "sha512-WFj2isz22JahUv+B788TlO3N6zL3nNJGU8CcZbPZvVEkBPaJdCV4vy5wyghty5ROFbCRnm132v8BScu5/1BQ8g==", "dev": true, "requires": { "debug": "^3.2.7", - "is-core-module": "^2.11.0", - "resolve": "^1.22.1" + "is-core-module": "^2.13.0", + "resolve": "^1.22.4" }, "dependencies": { "debug": { @@ -15050,9 +15805,9 @@ } }, "eslint-module-utils": { - "version": "2.7.4", - "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.7.4.tgz", - "integrity": "sha512-j4GT+rqzCoRKHwURX7pddtIPGySnX9Si/cgMI5ztrcqOPtk5dDEeZ34CQVPphnqkJytlc97Vuk05Um2mJ3gEQA==", + "version": "2.8.0", + "resolved": "https://registry.npmjs.org/eslint-module-utils/-/eslint-module-utils-2.8.0.tgz", + "integrity": "sha512-aWajIYfsqCKRDgUfjEXNN/JlrzauMuSEy5sbd7WXbtW3EH6A6MpwEh42c7qD+MqQo9QMJ6fWLAeIJynx0g6OAw==", "dev": true, "requires": { "debug": "^3.2.7" @@ -15097,26 +15852,28 @@ } }, "eslint-plugin-import": { - "version": "2.27.5", - "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.27.5.tgz", - "integrity": "sha512-LmEt3GVofgiGuiE+ORpnvP+kAm3h6MLZJ4Q5HCyHADofsb4VzXFsRiWj3c0OFiV+3DWFh0qg3v9gcPlfc3zRow==", + "version": "2.29.1", + "resolved": "https://registry.npmjs.org/eslint-plugin-import/-/eslint-plugin-import-2.29.1.tgz", + "integrity": "sha512-BbPC0cuExzhiMo4Ff1BTVwHpjjv28C5R+btTOGaCRC7UEz801up0JadwkeSk5Ued6TG34uaczuVuH6qyy5YUxw==", "dev": true, "requires": { - "array-includes": "^3.1.6", - "array.prototype.flat": "^1.3.1", - "array.prototype.flatmap": "^1.3.1", + "array-includes": "^3.1.7", + "array.prototype.findlastindex": "^1.2.3", + "array.prototype.flat": "^1.3.2", + "array.prototype.flatmap": "^1.3.2", "debug": "^3.2.7", "doctrine": "^2.1.0", - "eslint-import-resolver-node": "^0.3.7", - "eslint-module-utils": "^2.7.4", - "has": "^1.0.3", - "is-core-module": "^2.11.0", + "eslint-import-resolver-node": "^0.3.9", + "eslint-module-utils": "^2.8.0", + "hasown": "^2.0.0", + "is-core-module": "^2.13.1", "is-glob": "^4.0.3", "minimatch": "^3.1.2", - "object.values": "^1.1.6", - "resolve": "^1.22.1", - "semver": "^6.3.0", - "tsconfig-paths": "^3.14.1" + "object.fromentries": "^2.0.7", + "object.groupby": "^1.0.1", + "object.values": "^1.1.7", + "semver": "^6.3.1", + "tsconfig-paths": "^3.15.0" }, "dependencies": { "debug": { @@ -15140,36 +15897,29 @@ } }, "eslint-plugin-jsx-a11y": { - "version": "6.6.1", - "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.6.1.tgz", - "integrity": "sha512-sXgFVNHiWffBq23uiS/JaP6eVR622DqwB4yTzKvGZGcPq6/yZ3WmOZfuBks/vHWo9GaFOqC2ZK4i6+C35knx7Q==", - "dev": true, - "requires": { - "@babel/runtime": "^7.18.9", - "aria-query": "^4.2.2", - "array-includes": "^3.1.5", - "ast-types-flow": "^0.0.7", - "axe-core": "^4.4.3", - "axobject-query": "^2.2.0", + "version": "6.8.0", + "resolved": "https://registry.npmjs.org/eslint-plugin-jsx-a11y/-/eslint-plugin-jsx-a11y-6.8.0.tgz", + "integrity": "sha512-Hdh937BS3KdwwbBaKd5+PLCOmYY6U4f2h9Z2ktwtNKvIdIEu137rjYbcb9ApSbVJfWxANNuiKTD/9tOKjK9qOA==", + "dev": true, + "requires": { + "@babel/runtime": "^7.23.2", + "aria-query": "^5.3.0", + "array-includes": "^3.1.7", + "array.prototype.flatmap": "^1.3.2", + "ast-types-flow": "^0.0.8", + "axe-core": "=4.7.0", + "axobject-query": "^3.2.1", "damerau-levenshtein": "^1.0.8", "emoji-regex": "^9.2.2", - "has": "^1.0.3", - "jsx-ast-utils": "^3.3.2", - "language-tags": "^1.0.5", + "es-iterator-helpers": "^1.0.15", + "hasown": "^2.0.0", + "jsx-ast-utils": "^3.3.5", + "language-tags": "^1.0.9", "minimatch": "^3.1.2", - "semver": "^6.3.0" + "object.entries": "^1.1.7", + "object.fromentries": "^2.0.7" }, "dependencies": { - "aria-query": { - "version": "4.2.2", - "resolved": "https://registry.npmjs.org/aria-query/-/aria-query-4.2.2.tgz", - "integrity": "sha512-o/HelwhuKpTj/frsOsbNLNgnNGVIFsVP/SW2BSF14gVl7kAfMOJ6/8wUAUvG1R1NHKrfG+2sHZTu0yauT1qBrA==", - "dev": true, - "requires": { - "@babel/runtime": "^7.10.2", - "@babel/runtime-corejs3": "^7.10.2" - } - }, "emoji-regex": { "version": "9.2.2", "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", @@ -15213,15 +15963,16 @@ "requires": {} }, "eslint-plugin-react": { - "version": "7.33.0", - "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.33.0.tgz", - "integrity": "sha512-qewL/8P34WkY8jAqdQxsiL82pDUeT7nhs8IsuXgfgnsEloKCT4miAV9N9kGtx7/KM9NH/NCGUE7Edt9iGxLXFw==", + "version": "7.33.2", + "resolved": "https://registry.npmjs.org/eslint-plugin-react/-/eslint-plugin-react-7.33.2.tgz", + "integrity": "sha512-73QQMKALArI8/7xGLNI/3LylrEYrlKZSb5C9+q3OtOewTnMQi5cT+aE9E41sLCmli3I9PGGmD1yiZydyo4FEPw==", "dev": true, "requires": { "array-includes": "^3.1.6", "array.prototype.flatmap": "^1.3.1", "array.prototype.tosorted": "^1.1.1", "doctrine": "^2.1.0", + "es-iterator-helpers": "^1.0.12", "estraverse": "^5.3.0", "jsx-ast-utils": "^2.4.1 || ^3.0.0", "minimatch": "^3.1.2", @@ -15667,6 +16418,24 @@ "is-callable": "^1.1.3" } }, + "foreground-child": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/foreground-child/-/foreground-child-3.1.1.tgz", + "integrity": "sha512-TMKDUnIte6bfb5nWv7V/caI169OHgvwjb7V4WkeUvbQQdjr5rWKqHFiKWb/fcOwB+CzBT+qbWjvj+DVwRskpIg==", + "dev": true, + "requires": { + "cross-spawn": "^7.0.0", + "signal-exit": "^4.0.1" + }, + "dependencies": { + "signal-exit": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/signal-exit/-/signal-exit-4.1.0.tgz", + "integrity": "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw==", + "dev": true + } + } + }, "form-data": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.0.tgz", @@ -15758,9 +16527,9 @@ "devOptional": true }, "function-bind": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", - "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, "function.prototype.name": { "version": "1.1.5", @@ -15804,15 +16573,15 @@ "peer": true }, "get-intrinsic": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.1.tgz", - "integrity": "sha512-2DcsyfABl+gVHEfCOaTrWgyt+tb6MSEGmKq+kI5HwLbIYgjgmMcV8KQ41uaKz1xxUcn9tJtgFbQUEVcEbd0FYw==", + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.2.tgz", + "integrity": "sha512-0gSo4ml/0j98Y3lngkFEot/zhiCeWsbYIlZ+uZOVgzLyLaUw7wxUL+nCTP0XJvJg1AXulJRI3UJi8GsbDuxdGA==", "dev": true, "requires": { - "function-bind": "^1.1.1", - "has": "^1.0.3", + "function-bind": "^1.1.2", "has-proto": "^1.0.1", - "has-symbols": "^1.0.3" + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" } }, "get-nonce": { @@ -15868,11 +16637,6 @@ "is-glob": "^4.0.3" } }, - "glob-to-regexp": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz", - "integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw==" - }, "globals": { "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", @@ -15911,9 +16675,9 @@ } }, "graceful-fs": { - "version": "4.2.10", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.10.tgz", - "integrity": "sha512-9ByhssR2fPVsNZj478qUUbKfmL0+t5BDVyjShtyZZLiK7ZDAArFFfopyOTj0M05wE2tJPisA4iTnnXl2YoPvOA==" + "version": "4.2.11", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz", + "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==" }, "graphemer": { "version": "1.4.0", @@ -15925,6 +16689,7 @@ "version": "1.0.3", "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dev": true, "requires": { "function-bind": "^1.1.1" } @@ -15941,12 +16706,12 @@ "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==" }, "has-property-descriptors": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.0.tgz", - "integrity": "sha512-62DVLZGoiEBDHQyqG4w9xCuZ7eJEwNmJRWw2VY84Oedb7WFcA27fiEVe8oUQx9hAUJ4ekurquucTGwsyO1XGdQ==", + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.1.tgz", + "integrity": "sha512-VsX8eaIewvas0xnvinAe9bw4WfIeODpGYikiWYLH+dma0Jw6KHYqWiWfhQlgOVK8D6PvjubK5Uc4P0iIhIcNVg==", "dev": true, "requires": { - "get-intrinsic": "^1.1.1" + "get-intrinsic": "^1.2.2" } }, "has-proto": { @@ -15976,6 +16741,14 @@ "integrity": "sha512-8Rf9Y83NBReMnx0gFzA8JImQACstCYWUplepDa9xprwwtmgEZUF0h/i5xSA625zB/I37EtrswSST6OXxwaaIJQ==", "optional": true }, + "hasown": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.0.tgz", + "integrity": "sha512-vUptKVTpIJhcczKBbgnS+RtcuYMB8+oNzPK2/Hp3hanz8JmpATdmmgLgSaadVREkDm+e2giHwY3ZRkyjSIDDFA==", + "requires": { + "function-bind": "^1.1.2" + } + }, "hast-util-parse-selector": { "version": "2.2.5", "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-2.2.5.tgz", @@ -16178,6 +16951,15 @@ "resolved": "https://registry.npmjs.org/is-arrayish/-/is-arrayish-0.2.1.tgz", "integrity": "sha512-zz06S8t0ozoDXMG+ube26zeCTNXcKIPJZJi8hBrF4idCLms4CG9QtK7qBl1boi5ODzFpjswb5JPmHCbMpjaYzg==" }, + "is-async-function": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-async-function/-/is-async-function-2.0.0.tgz", + "integrity": "sha512-Y1JXKrfykRJGdlDwdKlLpLyMIiWqWvuSd17TvZk68PLAOGOoF4Xyav1z0Xhoi+gCYjZVeC5SI+hYFOfvXmGRCA==", + "dev": true, + "requires": { + "has-tostringtag": "^1.0.0" + } + }, "is-bigint": { "version": "1.0.4", "resolved": "https://registry.npmjs.org/is-bigint/-/is-bigint-1.0.4.tgz", @@ -16204,11 +16986,11 @@ "dev": true }, "is-core-module": { - "version": "2.12.1", - "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.12.1.tgz", - "integrity": "sha512-Q4ZuBAe2FUsKtyQJoQHlvP8OvBERxO3jEmy1I7hcRXcJBGGHFh/aJBswbXuS9sgrDH2QUO8ilkwNPHvHMd8clg==", + "version": "2.13.1", + "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", + "integrity": "sha512-hHrIjvZsftOsvKSn2TRYl63zvxsgE0K+0mYMoH6gD4omR5IWB2KynivBQczo3+wF1cCkjzvptnI9Q0sPU66ilw==", "requires": { - "has": "^1.0.3" + "hasown": "^2.0.0" } }, "is-date-object": { @@ -16237,11 +17019,29 @@ "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", "dev": true }, + "is-finalizationregistry": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/is-finalizationregistry/-/is-finalizationregistry-1.0.2.tgz", + "integrity": "sha512-0by5vtUJs8iFQb5TYUHHPudOR+qXYIMKtiUzvLIZITZUjknFmziyBJuLhVRc+Ds0dREFlskDNJKYIdIzu/9pfw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2" + } + }, "is-fullwidth-code-point": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz", "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", - "optional": true + "devOptional": true + }, + "is-generator-function": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", + "integrity": "sha512-jsEjy9l3yiXEQ+PsXdmBwEPcOxaXWLspKdplFUVI9vq1iZgIekeC0L167qeu86czQaxed3q/Uzuw0swL0irL8A==", + "dev": true, + "requires": { + "has-tostringtag": "^1.0.0" + } }, "is-glob": { "version": "4.0.3", @@ -16266,6 +17066,12 @@ "is-docker": "^3.0.0" } }, + "is-map": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-map/-/is-map-2.0.2.tgz", + "integrity": "sha512-cOZFQQozTha1f4MxLFzlgKYPTyj26picdZTx82hbc/Xf4K/tZOOXSCkMvU4pKioRXGDLJRn0GM7Upe7kR721yg==", + "dev": true + }, "is-negative-zero": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/is-negative-zero/-/is-negative-zero-2.0.2.tgz", @@ -16309,6 +17115,12 @@ "has-tostringtag": "^1.0.0" } }, + "is-set": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-set/-/is-set-2.0.2.tgz", + "integrity": "sha512-+2cnTEZeY5z/iXGbLhPrOAaK/Mau5k5eXq9j14CpRTftq0pAJu2MwVRSZhyZWBzx3o6X795Lz6Bpb6R0GKf37g==", + "dev": true + }, "is-shared-array-buffer": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-shared-array-buffer/-/is-shared-array-buffer-1.0.2.tgz", @@ -16351,6 +17163,12 @@ "which-typed-array": "^1.1.11" } }, + "is-weakmap": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/is-weakmap/-/is-weakmap-2.0.1.tgz", + "integrity": "sha512-NSBR4kH5oVj1Uwvv970ruUkCV7O1mzgVFO4/rev2cLRda9Tm9HrL70ZPut4rOHgY0FNrUu9BCbXA2sdQ+x0chA==", + "dev": true + }, "is-weakref": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/is-weakref/-/is-weakref-1.0.2.tgz", @@ -16360,6 +17178,16 @@ "call-bind": "^1.0.2" } }, + "is-weakset": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/is-weakset/-/is-weakset-2.0.2.tgz", + "integrity": "sha512-t2yVvttHkQktwnNNmBQ98AhENLdPUTDTE21uPqAQ0ARwQfGeQKRVS0NNurH7bTf7RrvcVn1OOge45CnBeHCSmg==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "get-intrinsic": "^1.1.1" + } + }, "is-wsl": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/is-wsl/-/is-wsl-2.2.0.tgz", @@ -16389,6 +17217,29 @@ "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==", "dev": true }, + "iterator.prototype": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/iterator.prototype/-/iterator.prototype-1.1.2.tgz", + "integrity": "sha512-DR33HMMr8EzwuRL8Y9D3u2BMj8+RqSE850jfGu59kS7tbmPLzGkZmVSfyCFSDxuZiEY6Rzt3T2NA/qU+NwVj1w==", + "dev": true, + "requires": { + "define-properties": "^1.2.1", + "get-intrinsic": "^1.2.1", + "has-symbols": "^1.0.3", + "reflect.getprototypeof": "^1.0.4", + "set-function-name": "^2.0.1" + } + }, + "jackspeak": { + "version": "2.3.6", + "resolved": "https://registry.npmjs.org/jackspeak/-/jackspeak-2.3.6.tgz", + "integrity": "sha512-N3yCS/NegsOBokc8GAdM8UcmfsKiSS8cipheD/nivzr700H+nsMOxJjQnvwOcRYVuFkdH0wGUvW2WbXGmrZGbQ==", + "dev": true, + "requires": { + "@isaacs/cliui": "^8.0.2", + "@pkgjs/parseargs": "^0.11.0" + } + }, "javascript-lp-solver": { "version": "0.4.24", "resolved": "https://registry.npmjs.org/javascript-lp-solver/-/javascript-lp-solver-0.4.24.tgz", @@ -16794,13 +17645,15 @@ "peer": true }, "jsx-ast-utils": { - "version": "3.3.3", - "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.3.tgz", - "integrity": "sha512-fYQHZTZ8jSfmWZ0iyzfwiU4WDX4HpHbMCZ3gPlWYiCl3BoeOTsqKBqnTVfH2rYT7eP5c3sVbeSPHnnJOaTrWiw==", + "version": "3.3.5", + "resolved": "https://registry.npmjs.org/jsx-ast-utils/-/jsx-ast-utils-3.3.5.tgz", + "integrity": "sha512-ZZow9HBI5O6EPgSJLUb8n2NKgmVWTwCvHGwFuJlMjvLFqlGG6pjirPhtdsseaLZjSibD8eegzmYpUZwoIlj2cQ==", "dev": true, "requires": { - "array-includes": "^3.1.5", - "object.assign": "^4.1.3" + "array-includes": "^3.1.6", + "array.prototype.flat": "^1.3.1", + "object.assign": "^4.1.4", + "object.values": "^1.1.6" } }, "language-subtag-registry": { @@ -16810,12 +17663,12 @@ "dev": true }, "language-tags": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-1.0.5.tgz", - "integrity": "sha512-qJhlO9cGXi6hBGKoxEG/sKZDAHD5Hnu9Hs4WbOY3pCWXDhw0N8x1NenNzm2EnNLkLkk7J2SdxAkDSbb6ftT+UQ==", + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/language-tags/-/language-tags-1.0.9.tgz", + "integrity": "sha512-MbjN408fEndfiQXbFQ1vnd+1NoLDsnQW41410oQBXiyXDMYH5z505juWa4KUE1LqxRC7DgOgZDbKLxHIwm27hA==", "dev": true, "requires": { - "language-subtag-registry": "~0.3.2" + "language-subtag-registry": "^0.3.20" } }, "levn": { @@ -17003,9 +17856,9 @@ } }, "minimist": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.6.tgz", - "integrity": "sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==", + "version": "1.2.8", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", + "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", "dev": true }, "minipass": { @@ -17079,26 +17932,26 @@ "dev": true }, "next": { - "version": "13.5.4", - "resolved": "https://registry.npmjs.org/next/-/next-13.5.4.tgz", - "integrity": "sha512-+93un5S779gho8y9ASQhb/bTkQF17FNQOtXLKAj3lsNgltEcF0C5PMLLncDmH+8X1EnJH1kbqAERa29nRXqhjA==", - "requires": { - "@next/env": "13.5.4", - "@next/swc-darwin-arm64": "13.5.4", - "@next/swc-darwin-x64": "13.5.4", - "@next/swc-linux-arm64-gnu": "13.5.4", - "@next/swc-linux-arm64-musl": "13.5.4", - "@next/swc-linux-x64-gnu": "13.5.4", - "@next/swc-linux-x64-musl": "13.5.4", - "@next/swc-win32-arm64-msvc": "13.5.4", - "@next/swc-win32-ia32-msvc": "13.5.4", - "@next/swc-win32-x64-msvc": "13.5.4", + "version": "14.1.0", + "resolved": "https://registry.npmjs.org/next/-/next-14.1.0.tgz", + "integrity": "sha512-wlzrsbfeSU48YQBjZhDzOwhWhGsy+uQycR8bHAOt1LY1bn3zZEcDyHQOEoN3aWzQ8LHCAJ1nqrWCc9XF2+O45Q==", + "requires": { + "@next/env": "14.1.0", + "@next/swc-darwin-arm64": "14.1.0", + "@next/swc-darwin-x64": "14.1.0", + "@next/swc-linux-arm64-gnu": "14.1.0", + "@next/swc-linux-arm64-musl": "14.1.0", + "@next/swc-linux-x64-gnu": "14.1.0", + "@next/swc-linux-x64-musl": "14.1.0", + "@next/swc-win32-arm64-msvc": "14.1.0", + "@next/swc-win32-ia32-msvc": "14.1.0", + "@next/swc-win32-x64-msvc": "14.1.0", "@swc/helpers": "0.5.2", "busboy": "1.6.0", - "caniuse-lite": "^1.0.30001406", + "caniuse-lite": "^1.0.30001579", + "graceful-fs": "^4.2.11", "postcss": "8.4.31", - "styled-jsx": "5.1.1", - "watchpack": "2.4.0" + "styled-jsx": "5.1.1" } }, "node-fetch": { @@ -17218,25 +18071,37 @@ } }, "object.entries": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.6.tgz", - "integrity": "sha512-leTPzo4Zvg3pmbQ3rDK69Rl8GQvIqMWubrkxONG9/ojtFE2rD9fjMKfSI5BxW3osRH1m6VdzmqK8oAY9aT4x5w==", + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.entries/-/object.entries-1.1.7.tgz", + "integrity": "sha512-jCBs/0plmPsOnrKAfFQXRG2NFjlhZgjjcBLSmTnEhU8U6vVTsVe8ANeQJCHTl3gSsI4J+0emOoCgoKlmQPMgmA==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" } }, "object.fromentries": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.6.tgz", - "integrity": "sha512-VciD13dswC4j1Xt5394WR4MzmAQmlgN72phd/riNp9vtD7tp4QQWJ0R4wvclXcafgcYK8veHRed2W6XeGBvcfg==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/object.fromentries/-/object.fromentries-2.0.7.tgz", + "integrity": "sha512-UPbPHML6sL8PI/mOqPwsH4G6iyXcCGzLin8KvEPenOZN5lpCNBZZQ+V62vdjB1mQHrmqGQt5/OJzemUA+KJmEA==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" + } + }, + "object.groupby": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/object.groupby/-/object.groupby-1.0.1.tgz", + "integrity": "sha512-HqaQtqLnp/8Bn4GL16cj+CUYbnpe1bh0TtEaWvybszDG4tgxCJuRpV8VGuvNaI1fAnI4lUJzDG55MXcOH4JZcQ==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1" } }, "object.hasown": { @@ -17250,14 +18115,14 @@ } }, "object.values": { - "version": "1.1.6", - "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.6.tgz", - "integrity": "sha512-FVVTkD1vENCsAcwNs9k6jea2uHC/X0+JcjG8YA60FN5CMaJmG95wT9jek/xX9nornqGRrBkKtzuAu2wuHpKqvw==", + "version": "1.1.7", + "resolved": "https://registry.npmjs.org/object.values/-/object.values-1.1.7.tgz", + "integrity": "sha512-aU6xnDFYT3x17e/f0IiiwlGPTy2jzMySGfUB4fq6z7CV8l85CWHDk5ErhyhpfDHhrOMwGFhSQkhMGHaIotA6Ng==", "dev": true, "requires": { "call-bind": "^1.0.2", - "define-properties": "^1.1.4", - "es-abstract": "^1.20.4" + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1" } }, "on-finished": { @@ -17398,6 +18263,30 @@ "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", "integrity": "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw==" }, + "path-scurry": { + "version": "1.10.1", + "resolved": "https://registry.npmjs.org/path-scurry/-/path-scurry-1.10.1.tgz", + "integrity": "sha512-MkhCqzzBEpPvxxQ71Md0b1Kk51W01lrYvlMzSUaIzNsODdd7mqhiimSZlr+VegAz5Z6Vzt9Xg2ttE//XBhH3EQ==", + "dev": true, + "requires": { + "lru-cache": "^9.1.1 || ^10.0.0", + "minipass": "^5.0.0 || ^6.0.2 || ^7.0.0" + }, + "dependencies": { + "lru-cache": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-10.2.0.tgz", + "integrity": "sha512-2bIM8x+VAf6JT4bKAljS1qUWgMsqZRPGJS6FSahIMPVvctcNhyVp7AJu7quxOW9jwkryBReKZY5tY5JYv2n/7Q==", + "dev": true + }, + "minipass": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/minipass/-/minipass-7.0.4.tgz", + "integrity": "sha512-jYofLM5Dam9279rdkWzqHozUo4ybjdZmCsDHePy5V/PbBcVMiSZR97gmAy45aqi8CK1lG2ECd356FU86avfwUQ==", + "dev": true + } + } + }, "path-to-regexp": { "version": "0.1.7", "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz", @@ -17877,6 +18766,20 @@ } } }, + "reflect.getprototypeof": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/reflect.getprototypeof/-/reflect.getprototypeof-1.0.4.tgz", + "integrity": "sha512-ECkTw8TmJwW60lOTR+ZkODISW6RQ8+2CL3COqtiJKLd6MmB45hN51HprHFziKLGkAuTGQhBb91V8cy+KHlaCjw==", + "dev": true, + "requires": { + "call-bind": "^1.0.2", + "define-properties": "^1.2.0", + "es-abstract": "^1.22.1", + "get-intrinsic": "^1.2.1", + "globalthis": "^1.0.3", + "which-builtin-type": "^1.1.3" + } + }, "refractor": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/refractor/-/refractor-3.6.0.tgz", @@ -17895,9 +18798,9 @@ } }, "regenerator-runtime": { - "version": "0.13.9", - "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.13.9.tgz", - "integrity": "sha512-p3VT+cOEgxFsRRA9X4lkI1E+k2/CtnKtU4gcxyaCUreilL/vqI6CdZ3wxVUx3UOUg+gnUOQQcRI7BmSI656MYA==" + "version": "0.14.1", + "resolved": "https://registry.npmjs.org/regenerator-runtime/-/regenerator-runtime-0.14.1.tgz", + "integrity": "sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==" }, "regexp.prototype.flags": { "version": "1.5.0", @@ -17923,11 +18826,11 @@ "optional": true }, "resolve": { - "version": "1.22.1", - "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.1.tgz", - "integrity": "sha512-nBpuuYuY5jFsli/JIs1oldw6fOQCBioohqWZg/2hiaOybXOft4lonv85uDOKXdf8rhyK159cxU5cDcK/NKk8zw==", + "version": "1.22.8", + "resolved": "https://registry.npmjs.org/resolve/-/resolve-1.22.8.tgz", + "integrity": "sha512-oKWePCxqpd6FlLvGV1VU0x7bkPmmCNolxzjMf4NczoDnQcIWrAF+cPtZn5i6n+RfD2d9i0tzpKnG6Yk168yIyw==", "requires": { - "is-core-module": "^2.9.0", + "is-core-module": "^2.13.0", "path-parse": "^1.0.7", "supports-preserve-symlinks-flag": "^1.0.0" } @@ -18048,13 +18951,13 @@ "integrity": "sha512-PdhdWy89SiZogBLaw42zdeqtRJ//zFd2PgQavcICDUgJT5oW10QCRKbJ6bg4r0/UY2M6BWd5tkxuGFRvCkgfHQ==" }, "safe-array-concat": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.0.0.tgz", - "integrity": "sha512-9dVEFruWIsnie89yym+xWTAYASdpw3CJV7Li/6zBewGf9z2i1j31rP6jnY0pHEO4QZh6N0K11bFjWmdR8UGdPQ==", + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/safe-array-concat/-/safe-array-concat-1.1.0.tgz", + "integrity": "sha512-ZdQ0Jeb9Ofti4hbt5lX3T2JcAamT9hfzYU1MNB+z/jaEbB6wfFfPIR/zEORmZqobkCCJhSjodobH6WHNmJ97dg==", "dev": true, "requires": { - "call-bind": "^1.0.2", - "get-intrinsic": "^1.2.0", + "call-bind": "^1.0.5", + "get-intrinsic": "^1.2.2", "has-symbols": "^1.0.3", "isarray": "^2.0.5" } @@ -18166,6 +19069,30 @@ "integrity": "sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw==", "optional": true }, + "set-function-length": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.0.tgz", + "integrity": "sha512-4DBHDoyHlM1IRPGYcoxexgh67y4ueR53FKV1yyxwFMY7aCqcN/38M1+SwZ/qJQ8iLv7+ck385ot4CcisOAPT9w==", + "dev": true, + "requires": { + "define-data-property": "^1.1.1", + "function-bind": "^1.1.2", + "get-intrinsic": "^1.2.2", + "gopd": "^1.0.1", + "has-property-descriptors": "^1.0.1" + } + }, + "set-function-name": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/set-function-name/-/set-function-name-2.0.1.tgz", + "integrity": "sha512-tMNCiqYVkXIZgc2Hnoy2IvC/f8ezc5koaRFkCjrpWzGpCd3qbZXPzVy9MAZzK1ch/X0jvSkojys3oqJN0qCmdA==", + "dev": true, + "requires": { + "define-data-property": "^1.0.1", + "functions-have-names": "^1.2.3", + "has-property-descriptors": "^1.0.0" + } + }, "setimmediate": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", @@ -18306,7 +19233,18 @@ "version": "4.2.3", "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", - "optional": true, + "devOptional": true, + "requires": { + "emoji-regex": "^8.0.0", + "is-fullwidth-code-point": "^3.0.0", + "strip-ansi": "^6.0.1" + } + }, + "string-width-cjs": { + "version": "npm:string-width@4.2.3", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz", + "integrity": "sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==", + "dev": true, "requires": { "emoji-regex": "^8.0.0", "is-fullwidth-code-point": "^3.0.0", @@ -18371,6 +19309,15 @@ "ansi-regex": "^5.0.1" } }, + "strip-ansi-cjs": { + "version": "npm:strip-ansi@6.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz", + "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.1" + } + }, "strip-bom": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", @@ -18598,13 +19545,13 @@ } }, "tsconfig-paths": { - "version": "3.14.1", - "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.14.1.tgz", - "integrity": "sha512-fxDhWnFSLt3VuTwtvJt5fpwxBHg5AdKWMsgcPOOIilyjymcYVZoCQF8fvFRezCNfblEXmi+PcM1eYHeOAgXCOQ==", + "version": "3.15.0", + "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.15.0.tgz", + "integrity": "sha512-2Ac2RgzDe/cn48GvOe3M+o82pEFewD3UPbyoUHHdKasHwJKjds4fLXWf/Ux5kATBKN20oaFGu+jbElp1pos0mg==", "dev": true, "requires": { "@types/json5": "^0.0.29", - "json5": "^1.0.1", + "json5": "^1.0.2", "minimist": "^1.2.6", "strip-bom": "^3.0.0" }, @@ -18715,9 +19662,9 @@ } }, "typescript": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.1.6.tgz", - "integrity": "sha512-zaWCozRZ6DLEWAWFrVDz1H6FVXzUSfTy5FUMWsQlU8Ym5JP9eO4xkTIROFCQvhQf61z6O/G6ugw3SgAnvvm+HA==", + "version": "5.3.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", "dev": true }, "ua-parser-js": { @@ -18737,6 +19684,11 @@ "which-boxed-primitive": "^1.0.2" } }, + "undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" + }, "universalify": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.2.0.tgz", @@ -18878,15 +19830,6 @@ "xml-name-validator": "^4.0.0" } }, - "watchpack": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz", - "integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==", - "requires": { - "glob-to-regexp": "^0.4.1", - "graceful-fs": "^4.1.2" - } - }, "webidl-conversions": { "version": "7.0.0", "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-7.0.0.tgz", @@ -18940,6 +19883,38 @@ "is-symbol": "^1.0.3" } }, + "which-builtin-type": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/which-builtin-type/-/which-builtin-type-1.1.3.tgz", + "integrity": "sha512-YmjsSMDBYsM1CaFiayOVT06+KJeXf0o5M/CAd4o1lTadFAtacTUM49zoYxr/oroopFDfhvN6iEcBxUyc3gvKmw==", + "dev": true, + "requires": { + "function.prototype.name": "^1.1.5", + "has-tostringtag": "^1.0.0", + "is-async-function": "^2.0.0", + "is-date-object": "^1.0.5", + "is-finalizationregistry": "^1.0.2", + "is-generator-function": "^1.0.10", + "is-regex": "^1.1.4", + "is-weakref": "^1.0.2", + "isarray": "^2.0.5", + "which-boxed-primitive": "^1.0.2", + "which-collection": "^1.0.1", + "which-typed-array": "^1.1.9" + } + }, + "which-collection": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/which-collection/-/which-collection-1.0.1.tgz", + "integrity": "sha512-W8xeTUwaln8i3K/cY1nGXzdnVZlidBcagyNFtBdD5kxnb4TvGKR7FfSIS3mYpwWS1QUCutfKz8IY8RjftB0+1A==", + "dev": true, + "requires": { + "is-map": "^2.0.1", + "is-set": "^2.0.1", + "is-weakmap": "^2.0.1", + "is-weakset": "^2.0.1" + } + }, "which-typed-array": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/which-typed-array/-/which-typed-array-1.1.11.tgz", @@ -18968,6 +19943,94 @@ "integrity": "sha512-2V81OA4ugVo5pRo46hAoD2ivUJx8jXmWXfUkY4KFNw0hEptvN0QfH3K4nHiwzGeKl5rFKedV48QVoqYavy4YpA==", "optional": true }, + "wrap-ansi": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz", + "integrity": "sha512-si7QWI6zUMq56bESFvagtmzMdGOtoxfR+Sez11Mobfc7tm+VkUckk9bW2UeffTGVUbOksxmSw0AA2gs8g71NCQ==", + "dev": true, + "requires": { + "ansi-styles": "^6.1.0", + "string-width": "^5.0.1", + "strip-ansi": "^7.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true + }, + "ansi-styles": { + "version": "6.2.1", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-6.2.1.tgz", + "integrity": "sha512-bN798gFfQX+viw3R7yrGWRqnrN2oRkEkUjjl4JNn4E8GxxbjtG3FbrEIIY3l8/hrwUwIeCZvi4QuOTP4MErVug==", + "dev": true + }, + "emoji-regex": { + "version": "9.2.2", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-9.2.2.tgz", + "integrity": "sha512-L18DaJsXSUk2+42pv8mLs5jJT2hqFkFE4j21wOmgbUqsZ2hL72NsUU785g9RXgo3s0ZNgVl42TiHp3ZtOv/Vyg==", + "dev": true + }, + "string-width": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-5.1.2.tgz", + "integrity": "sha512-HnLOCR3vjcY8beoNLtcjZ5/nxn2afmME6lhrDrebokqMap+XbeW8n9TXpPDOqdGK5qcI3oT0GKTW6wC7EMiVqA==", + "dev": true, + "requires": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^9.2.2", + "strip-ansi": "^7.0.1" + } + }, + "strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "requires": { + "ansi-regex": "^6.0.1" + } + } + } + }, + "wrap-ansi-cjs": { + "version": "npm:wrap-ansi@7.0.0", + "resolved": "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz", + "integrity": "sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q==", + "dev": true, + "requires": { + "ansi-styles": "^4.0.0", + "string-width": "^4.1.0", + "strip-ansi": "^6.0.0" + }, + "dependencies": { + "ansi-styles": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-4.3.0.tgz", + "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", + "dev": true, + "requires": { + "color-convert": "^2.0.1" + } + }, + "color-convert": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz", + "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", + "dev": true, + "requires": { + "color-name": "~1.1.4" + } + }, + "color-name": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.4.tgz", + "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", + "dev": true + } + } + }, "wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/dashboard/package.json b/dashboard/package.json index adc87d007cfd6..1ac15145998dd 100644 --- a/dashboard/package.json +++ b/dashboard/package.json @@ -4,12 +4,13 @@ "dev": "next dev", "build": "next build", "start": "next start", - "build-static": "next build && next export", + "build-static": "next build", "gen-proto": "./scripts/generate_proto.sh", "lint": "prettier --check . && next lint", "format": "prettier --write . && next lint --fix", "mock-server": "node ./mock-server.js", - "postinstall": "npm run gen-proto" + "postinstall": "npm run gen-proto", + "clean": "rm -rf .next/ && rm -rf out/" }, "dependencies": { "@chakra-ui/react": "^2.3.1", @@ -32,7 +33,7 @@ "fabric": "^5.2.1", "framer-motion": "^6.5.1", "lodash": "^4.17.21", - "next": "^13.5.4", + "next": "^14.1.0", "nuqs": "^1.14.1", "react": "^18.2.0", "react-dom": "^18.2.0", @@ -47,12 +48,12 @@ "@testing-library/jest-dom": "^5.16.2", "@testing-library/react": "^13.3.0", "@types/loadable__component": "^5.13.4", - "@types/node": "^18.7.14", + "@types/node": "20.11.0", "@types/styled-components": "^5.1.26", "@typescript-eslint/eslint-plugin": "^5.52.0", "cors": "^2.8.5", "eslint": "^8.45.0", - "eslint-config-next": "13.4.12", + "eslint-config-next": "^14.1.0", "eslint-config-prettier": "^8.8.0", "eslint-config-standard-with-typescript": "^37.0.0", "eslint-plugin-import": "^2.27.5", @@ -62,7 +63,7 @@ "express": "^4.18.1", "prettier": "^2.7.1", "prettier-plugin-organize-imports": "^3.1.1", - "typescript": "^5.1.6" + "typescript": "^5.3.3" }, "overrides": { "react-json-view": { @@ -74,4 +75,4 @@ "react-dom": "^18.2.0" } } -} +} \ No newline at end of file diff --git a/dashboard/pages/api/metric.ts b/dashboard/pages/api/metric.ts deleted file mode 100644 index 2ea62cebc4385..0000000000000 --- a/dashboard/pages/api/metric.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2024 RisingWave Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * - */ -import { Metrics, MetricsSample } from "../../components/metrics" -import api from "./api" - -export interface BackPressuresMetrics { - outputBufferBlockingDuration: Metrics[] -} - -export async function getActorBackPressures() { - const res: BackPressuresMetrics = await api.get( - "/metrics/actor/back_pressures" - ) - return res -} - -function calculatePercentile(samples: MetricsSample[], percentile: number) { - const sorted = samples.sort((a, b) => a.value - b.value) - const index = Math.floor(sorted.length * percentile) - return sorted[index].value -} - -export function p50(samples: MetricsSample[]) { - return calculatePercentile(samples, 0.5) -} - -export function p90(samples: MetricsSample[]) { - return calculatePercentile(samples, 0.9) -} - -export function p95(samples: MetricsSample[]) { - return calculatePercentile(samples, 0.95) -} - -export function p99(samples: MetricsSample[]) { - return calculatePercentile(samples, 0.99) -} diff --git a/dashboard/pages/await_tree.tsx b/dashboard/pages/await_tree.tsx index 6fc899f4b7935..454a540d769e8 100644 --- a/dashboard/pages/await_tree.tsx +++ b/dashboard/pages/await_tree.tsx @@ -30,10 +30,10 @@ import Head from "next/head" import { Fragment, useEffect, useState } from "react" import SpinnerOverlay from "../components/SpinnerOverlay" import Title from "../components/Title" +import api from "../lib/api/api" +import { getClusterInfoComputeNode } from "../lib/api/cluster" +import useFetch from "../lib/api/fetch" import { StackTraceResponse } from "../proto/gen/monitor_service" -import api from "./api/api" -import { getClusterInfoComputeNode } from "./api/cluster" -import useFetch from "./api/fetch" const SIDEBAR_WIDTH = 200 const ALL_COMPUTE_NODES = "" diff --git a/dashboard/pages/cluster.tsx b/dashboard/pages/cluster.tsx index 5edc80d862dcd..eabc07741bcb1 100644 --- a/dashboard/pages/cluster.tsx +++ b/dashboard/pages/cluster.tsx @@ -32,12 +32,12 @@ import { Area, AreaChart, ResponsiveContainer, XAxis, YAxis } from "recharts" import { Metrics, MetricsSample } from "../components/metrics" import Title from "../components/Title" import useErrorToast from "../hook/useErrorToast" -import { WorkerNode } from "../proto/gen/common" import { getClusterInfoComputeNode, getClusterInfoFrontend, getClusterMetrics, -} from "./api/cluster" +} from "../lib/api/cluster" +import { WorkerNode } from "../proto/gen/common" function WorkerNodeComponent({ workerNodeType, diff --git a/dashboard/pages/dependency_graph.tsx b/dashboard/pages/dependency_graph.tsx index a4c13a94df169..96aeed23f8696 100644 --- a/dashboard/pages/dependency_graph.tsx +++ b/dashboard/pages/dependency_graph.tsx @@ -24,9 +24,13 @@ import RelationDependencyGraph, { nodeRadius, } from "../components/RelationDependencyGraph" import Title from "../components/Title" +import useFetch from "../lib/api/fetch" +import { + Relation, + getRelations, + relationIsStreamingJob, +} from "../lib/api/streaming" import { RelationPoint } from "../lib/layout" -import useFetch from "./api/fetch" -import { Relation, getRelations, relationIsStreamingJob } from "./api/streaming" const SIDEBAR_WIDTH = "200px" diff --git a/dashboard/pages/fragment_graph.tsx b/dashboard/pages/fragment_graph.tsx index 244b78a540a0a..e72ca7ce3e0ce 100644 --- a/dashboard/pages/fragment_graph.tsx +++ b/dashboard/pages/fragment_graph.tsx @@ -37,17 +37,25 @@ import FragmentDependencyGraph from "../components/FragmentDependencyGraph" import FragmentGraph from "../components/FragmentGraph" import Title from "../components/Title" import useErrorToast from "../hook/useErrorToast" +import useFetch from "../lib/api/fetch" +import { + BackPressureInfo, + calculateBPRate, + fetchEmbeddedBackPressure, + fetchPrometheusBackPressure, +} from "../lib/api/metric" +import { getFragments, getStreamingJobs } from "../lib/api/streaming" import { FragmentBox } from "../lib/layout" import { TableFragments, TableFragments_Fragment } from "../proto/gen/meta" import { Dispatcher, MergeNode, StreamNode } from "../proto/gen/stream_plan" -import useFetch from "./api/fetch" -import { getActorBackPressures, p50, p90, p95, p99 } from "./api/metric" -import { getFragments, getStreamingJobs } from "./api/streaming" interface DispatcherNode { [actorId: number]: Dispatcher[] } +// Refresh interval (ms) for back pressure stats +const INTERVAL = 5000 + /** Associated data of each plan node in the fragment graph, including the dispatchers. */ export interface PlanNodeDatum { name: string @@ -168,22 +176,27 @@ function buildFragmentDependencyAsEdges( const SIDEBAR_WIDTH = 200 -type BackPressureAlgo = "p50" | "p90" | "p95" | "p99" -const backPressureAlgos: BackPressureAlgo[] = ["p50", "p90", "p95", "p99"] +type BackPressureDataSource = "Embedded" | "Prometheus" +const backPressureDataSources: BackPressureDataSource[] = [ + "Embedded", + "Prometheus", +] + +// The state of the embedded back pressure metrics. +// The metrics from previous fetch are stored here to calculate the rate. +interface EmbeddedBackPressureInfo { + previous: BackPressureInfo[] + current: BackPressureInfo[] +} export default function Streaming() { const { response: relationList } = useFetch(getStreamingJobs) const { response: fragmentList } = useFetch(getFragments) const [relationId, setRelationId] = useQueryState("id", parseAsInteger) - const [backPressureAlgo, setBackPressureAlgo] = useQueryState("backPressure") const [selectedFragmentId, setSelectedFragmentId] = useState() - const { response: actorBackPressures } = useFetch( - getActorBackPressures, - 5000, - backPressureAlgo !== null - ) + const toast = useErrorToast() const fragmentDependencyCallback = useCallback(() => { if (fragmentList) { @@ -199,7 +212,6 @@ export default function Streaming() { } } } - return undefined }, [fragmentList, relationId]) useEffect(() => { @@ -210,7 +222,6 @@ export default function Streaming() { } } } - return () => {} }, [relationId, relationList, setRelationId]) const fragmentDependency = fragmentDependencyCallback()?.fragmentDep @@ -239,8 +250,6 @@ export default function Streaming() { const [searchActorId, setSearchActorId] = useState("") const [searchFragId, setSearchFragId] = useState("") - const toast = useErrorToast() - const handleSearchFragment = () => { const searchFragIdInt = parseInt(searchFragId) if (fragmentList) { @@ -278,39 +287,86 @@ export default function Streaming() { toast(new Error(`Actor ${searchActorIdInt} not found`)) } + const [backPressureDataSource, setBackPressureDataSource] = + useState("Embedded") + + // Periodically fetch Prometheus back-pressure from Meta node + const { response: promethusMetrics } = useFetch( + fetchPrometheusBackPressure, + INTERVAL, + backPressureDataSource === "Prometheus" + ) + + // Periodically fetch embedded back-pressure from Meta node + // Didn't call `useFetch()` because the `setState` way is special. + const [embeddedBackPressureInfo, setEmbeddedBackPressureInfo] = + useState() + useEffect(() => { + if (backPressureDataSource === "Embedded") { + const interval = setInterval(() => { + fetchEmbeddedBackPressure().then( + (newBP) => { + console.log(newBP) + setEmbeddedBackPressureInfo((prev) => + prev + ? { + previous: prev.current, + current: newBP, + } + : { + previous: newBP, // Use current value to show zero rate, but it's fine + current: newBP, + } + ) + }, + (e) => { + console.error(e) + toast(e, "error") + } + ) + }, INTERVAL) + return () => { + clearInterval(interval) + } + } + }, [backPressureDataSource]) + const backPressures = useMemo(() => { - if (actorBackPressures && backPressureAlgo) { + if (promethusMetrics || embeddedBackPressureInfo) { let map = new Map() - for (const m of actorBackPressures.outputBufferBlockingDuration) { - console.log(backPressureAlgo) - let algoFunc - switch (backPressureAlgo) { - case "p50": - algoFunc = p50 - break - case "p90": - algoFunc = p90 - break - case "p95": - algoFunc = p95 - break - case "p99": - algoFunc = p99 - break - default: - return - } - - const value = algoFunc(m.sample) * 100 - map.set( - `${m.metric.fragment_id}_${m.metric.downstream_fragment_id}`, - value + if (backPressureDataSource === "Embedded" && embeddedBackPressureInfo) { + const metrics = calculateBPRate( + embeddedBackPressureInfo.current, + embeddedBackPressureInfo.previous, + INTERVAL ) + for (const m of metrics.outputBufferBlockingDuration) { + map.set( + `${m.metric.fragmentId}_${m.metric.downstreamFragmentId}`, + m.sample[0].value + ) + } + } else if (backPressureDataSource === "Prometheus" && promethusMetrics) { + for (const m of promethusMetrics.outputBufferBlockingDuration) { + if (m.sample.length > 0) { + // Note: We issue an instant query to Prometheus to get the most recent value. + // So there should be only one sample here. + // + // Due to https://github.com/risingwavelabs/risingwave/issues/15280, it's still + // possible that an old version of meta service returns a range-query result. + // So we take the one with the latest timestamp here. + const value = _(m.sample).maxBy((s) => s.timestamp)!.value * 100 + map.set( + `${m.metric.fragment_id}_${m.metric.downstream_fragment_id}`, + value + ) + } + } } return map } - }, [actorBackPressures, backPressureAlgo]) + }, [backPressureDataSource, promethusMetrics, embeddedBackPressureInfo]) const retVal = ( @@ -381,23 +437,22 @@ export default function Streaming() { - Back Pressure + Back Pressure Data Source diff --git a/dashboard/pages/heap_profiling.tsx b/dashboard/pages/heap_profiling.tsx index 11098a19ab705..ffed780ee2022 100644 --- a/dashboard/pages/heap_profiling.tsx +++ b/dashboard/pages/heap_profiling.tsx @@ -32,11 +32,11 @@ import path from "path" import { Fragment, useEffect, useState } from "react" import SpinnerOverlay from "../components/SpinnerOverlay" import Title from "../components/Title" +import api from "../lib/api/api" +import { getClusterInfoComputeNode } from "../lib/api/cluster" +import useFetch from "../lib/api/fetch" import { WorkerNode } from "../proto/gen/common" import { ListHeapProfilingResponse } from "../proto/gen/monitor_service" -import api from "./api/api" -import { getClusterInfoComputeNode } from "./api/cluster" -import useFetch from "./api/fetch" const SIDEBAR_WIDTH = 200 diff --git a/dashboard/pages/indexes.tsx b/dashboard/pages/indexes.tsx index daee8df63a556..0e57e724dc206 100644 --- a/dashboard/pages/indexes.tsx +++ b/dashboard/pages/indexes.tsx @@ -20,7 +20,7 @@ import { Relations, streamingJobColumns, } from "../components/Relations" -import { getIndexes } from "./api/streaming" +import { getIndexes } from "../lib/api/streaming" export default function Indexes() { return Relations("Indexes", getIndexes, [ diff --git a/dashboard/pages/internal_tables.tsx b/dashboard/pages/internal_tables.tsx index 21650dc282e12..2bb0c66bf9c6c 100644 --- a/dashboard/pages/internal_tables.tsx +++ b/dashboard/pages/internal_tables.tsx @@ -16,7 +16,7 @@ */ import { primaryKeyColumn, Relations } from "../components/Relations" -import { getInternalTables } from "./api/streaming" +import { getInternalTables } from "../lib/api/streaming" export default function InternalTables() { return Relations("Internal Tables", getInternalTables, [primaryKeyColumn]) diff --git a/dashboard/pages/materialized_views.tsx b/dashboard/pages/materialized_views.tsx index 7f6f27ab4afc4..d1bde8ac6c582 100644 --- a/dashboard/pages/materialized_views.tsx +++ b/dashboard/pages/materialized_views.tsx @@ -20,7 +20,7 @@ import { Relations, streamingJobColumns, } from "../components/Relations" -import { getMaterializedViews } from "./api/streaming" +import { getMaterializedViews } from "../lib/api/streaming" export default function MaterializedViews() { return Relations("Materialized Views", getMaterializedViews, [ diff --git a/dashboard/pages/settings.tsx b/dashboard/pages/settings.tsx index 3214bbdf5c746..121e1f0bd7250 100644 --- a/dashboard/pages/settings.tsx +++ b/dashboard/pages/settings.tsx @@ -24,7 +24,7 @@ import { API_ENDPOINT_KEY, DEFAULT_API_ENDPOINT, PREDEFINED_API_ENDPOINTS, -} from "./api/api" +} from "../lib/api/api" export default function Settings() { const isClient = useIsClient() diff --git a/dashboard/pages/sinks.tsx b/dashboard/pages/sinks.tsx index 825a3a7af2220..60c6828c0c0b0 100644 --- a/dashboard/pages/sinks.tsx +++ b/dashboard/pages/sinks.tsx @@ -20,7 +20,7 @@ import { Relations, streamingJobColumns, } from "../components/Relations" -import { getSinks } from "./api/streaming" +import { getSinks } from "../lib/api/streaming" export default function Sinks() { return Relations("Sinks", getSinks, [ diff --git a/dashboard/pages/sources.tsx b/dashboard/pages/sources.tsx index 898dd32c7f890..84b4edc30bf14 100644 --- a/dashboard/pages/sources.tsx +++ b/dashboard/pages/sources.tsx @@ -21,8 +21,8 @@ import { dependentsColumn, Relations, } from "../components/Relations" +import { getSources } from "../lib/api/streaming" import { Source } from "../proto/gen/catalog" -import { getSources } from "./api/streaming" export default function DataSources() { const rowFormatColumn: Column = { diff --git a/dashboard/pages/tables.tsx b/dashboard/pages/tables.tsx index 48af41593629a..c74d5b8ad4754 100644 --- a/dashboard/pages/tables.tsx +++ b/dashboard/pages/tables.tsx @@ -21,8 +21,8 @@ import { Relations, streamingJobColumns, } from "../components/Relations" +import { getTables } from "../lib/api/streaming" import { Table } from "../proto/gen/catalog" -import { getTables } from "./api/streaming" export default function Tables() { const associatedSourceColumn: Column = { diff --git a/dashboard/pages/views.tsx b/dashboard/pages/views.tsx index bb6d3e4e6b6fd..ad8bd16e4a602 100644 --- a/dashboard/pages/views.tsx +++ b/dashboard/pages/views.tsx @@ -16,7 +16,7 @@ */ import { Relations } from "../components/Relations" -import { getViews } from "./api/streaming" +import { getViews } from "../lib/api/streaming" export default function Views() { return Relations("Views", getViews, []) diff --git a/docker/Dockerfile b/docker/Dockerfile index 58e93de54bcef..d42dc74da3f11 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -5,6 +5,13 @@ ENV LANG en_US.utf8 RUN apt-get update \ && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk +# Install Python 3.12 +RUN apt-get -y install software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa -y && \ + apt-get update -yy && \ + DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy +ENV PYO3_PYTHON=python3.12 + FROM base AS dashboard-builder RUN apt-get update && apt-get install -y curl gnupg protobuf-compiler && mkdir -p /etc/apt/keyrings \ @@ -33,7 +40,8 @@ COPY rust-toolchain rust-toolchain RUN rustup self update \ && rustup set profile minimal \ && rustup show \ - && rustup component add rustfmt + && rustup component add rustfmt \ + && rustup target add wasm32-wasi RUN cargo install flamegraph # TODO: cargo-chef doesn't work well now, because we update Cargo.lock very often. @@ -69,7 +77,7 @@ COPY ./ /risingwave WORKDIR /risingwave RUN cargo fetch && \ - cargo build -p risingwave_cmd_all --release --features "rw-static-link" && \ + cargo build -p risingwave_cmd_all --release --features "rw-static-link" --features embedded-python-udf && \ mkdir -p /risingwave/bin && \ mv /risingwave/target/release/risingwave /risingwave/bin/ && \ mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \ diff --git a/docker/Dockerfile.hdfs b/docker/Dockerfile.hdfs index e8dd1988bd6fe..28859a453c9a1 100644 --- a/docker/Dockerfile.hdfs +++ b/docker/Dockerfile.hdfs @@ -5,6 +5,13 @@ ENV LANG en_US.utf8 RUN apt-get update \ && apt-get -y install ca-certificates build-essential libsasl2-dev openjdk-11-jdk +# Install Python 3.12 +RUN apt-get -y install software-properties-common +RUN add-apt-repository ppa:deadsnakes/ppa -y && \ + apt-get update -yy && \ + DEBIAN_FRONTEND=noninteractive apt-get install python3.12 python3.12-dev -yy +ENV PYO3_PYTHON=python3.12 + FROM base AS builder RUN apt-get update && apt-get -y install make cmake protobuf-compiler curl bash lld maven unzip @@ -34,7 +41,8 @@ RUN unzip dashboard-artifact.zip && mv risingwave-dashboard-artifact /risingwave RUN rustup self update \ && rustup set profile minimal \ && rustup show \ - && rustup component add rustfmt + && rustup component add rustfmt \ + && rustup target add wasm32-wasi RUN cargo fetch @@ -44,7 +52,7 @@ ENV JAVA_HOME ${JAVA_HOME_PATH} ENV LD_LIBRARY_PATH ${JAVA_HOME_PATH}/lib/server:${LD_LIBRARY_PATH} RUN cargo fetch && \ - cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" && \ + cargo build -p risingwave_cmd_all --release -p risingwave_object_store --features hdfs-backend --features "rw-static-link" --features embedded-python-udf && \ mkdir -p /risingwave/bin && \ mv /risingwave/target/release/risingwave /risingwave/bin/ && \ mv /risingwave/target/release/risingwave.dwp /risingwave/bin/ && \ diff --git a/docker/dashboards/risingwave-dev-dashboard.json b/docker/dashboards/risingwave-dev-dashboard.json index 1db138e79999f..2fa9628478105 100644 --- a/docker/dashboards/risingwave-dev-dashboard.json +++ b/docker/dashboards/risingwave-dev-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - fragment_id {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":35,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":65,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":85,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, executor_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{executor_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink by Connector","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":90,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":96,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":117,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":156,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":184,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":204,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":207,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":211,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":214,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":218,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":223,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":231,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":240,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":252,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":267,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":285,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":293,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":299,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - fragment_id {{fragment_id}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_arrangement_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_arrangement_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Arrangement Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_arrangement_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_arrangement_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Arrangement Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":36,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":41,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":67,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":84,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":87,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, executor_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{executor_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, executor_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink by Connector","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":92,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":98,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":119,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":132,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":158,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":167,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","query":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":186,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":206,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":209,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":213,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":216,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":220,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":225,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":233,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":242,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":253,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":256,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":271,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":289,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":297,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","query":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":303,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} diff --git a/docker/dashboards/risingwave-user-dashboard.json b/docker/dashboards/risingwave-user-dashboard.json index d180e0e4b16c5..f9ec8cab9831f 100644 --- a/docker/dashboards/risingwave-user-dashboard.json +++ b/docker/dashboards/risingwave-user-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n - Write Stall: Compaction cannot keep up. Stall foreground write.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":42},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":50},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":51},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":53},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":54},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":55},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n - Write Stall: Compaction cannot keep up. Stall foreground write.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} diff --git a/docker/docker-compose-distributed.yml b/docker/docker-compose-distributed.yml index 8cb1e87651325..c1ca626a824e6 100644 --- a/docker/docker-compose-distributed.yml +++ b/docker/docker-compose-distributed.yml @@ -1,7 +1,7 @@ --- version: "3" x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.0} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.1} services: compactor-0: <<: *image @@ -213,6 +213,8 @@ services: - "0.0.0.0:5691" - "--prometheus-host" - "0.0.0.0:1250" + - "--prometheus-endpoint" + - "http://prometheus-0:9500" - "--backend" - etcd - "--etcd-endpoints" diff --git a/docker/docker-compose-with-azblob.yml b/docker/docker-compose-with-azblob.yml index e7149664ad23b..e43d28a96ffe5 100644 --- a/docker/docker-compose-with-azblob.yml +++ b/docker/docker-compose-with-azblob.yml @@ -1,7 +1,7 @@ --- version: "3" x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.0} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.1} services: risingwave-standalone: <<: *image @@ -10,6 +10,7 @@ services: --advertise-addr 0.0.0.0:5690 \ --dashboard-host 0.0.0.0:5691 \ --prometheus-host 0.0.0.0:1250 \ + --prometheus-endpoint http://prometheus-0:9500 \ --connector-rpc-endpoint 0.0.0.0:50051 \ --backend etcd \ --etcd-endpoints etcd-0:2388 \ @@ -19,7 +20,7 @@ services: --compute-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:5688 \ - --prometheus-listener-addr 0.0.0.0:1222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ --connector-rpc-endpoint 0.0.0.0:50051 \ @@ -31,32 +32,26 @@ services: --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ - --prometheus-listener-addr 0.0.0.0:2222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ --meta-addr http://0.0.0.0:5690\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ - --prometheus-listener-addr 0.0.0.0:1260 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ --meta-address http://0.0.0.0:5690\"" expose: - "6660" - - "1260" - "4566" - "5688" - - "1222" - "5690" - "1250" - "5691" - - "2222" ports: - "4566:4566" - "5690:5690" - "5691:5691" - - "1222:1222" - "1250:1250" - - "1260:1260" - - "2222:2222" depends_on: - etcd-0 env_file: multiple_object_storage.env diff --git a/docker/docker-compose-with-gcs.yml b/docker/docker-compose-with-gcs.yml index 45a5b3d17dce1..5300c6418581d 100644 --- a/docker/docker-compose-with-gcs.yml +++ b/docker/docker-compose-with-gcs.yml @@ -1,7 +1,7 @@ --- version: "3" x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.0} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.1} services: risingwave-standalone: <<: *image @@ -10,6 +10,7 @@ services: --advertise-addr 0.0.0.0:5690 \ --dashboard-host 0.0.0.0:5691 \ --prometheus-host 0.0.0.0:1250 \ + --prometheus-endpoint http://prometheus-0:9500 \ --connector-rpc-endpoint 0.0.0.0:50051 \ --backend etcd \ --etcd-endpoints etcd-0:2388 \ @@ -19,7 +20,7 @@ services: --compute-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:5688 \ - --prometheus-listener-addr 0.0.0.0:1222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ --connector-rpc-endpoint 0.0.0.0:50051 \ @@ -31,32 +32,26 @@ services: --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ - --prometheus-listener-addr 0.0.0.0:2222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ --meta-addr http://0.0.0.0:5690\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ - --prometheus-listener-addr 0.0.0.0:1260 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ --meta-address http://0.0.0.0:5690\"" expose: - "6660" - - "1260" - "4566" - "5688" - - "1222" - "5690" - "1250" - "5691" - - "2222" ports: - "4566:4566" - "5690:5690" - "5691:5691" - - "1222:1222" - "1250:1250" - - "1260:1260" - - "2222:2222" depends_on: - etcd-0 env_file: multiple_object_storage.env diff --git a/docker/docker-compose-with-hdfs.yml b/docker/docker-compose-with-hdfs.yml index ccf0f433c450b..cf2b45078bac5 100644 --- a/docker/docker-compose-with-hdfs.yml +++ b/docker/docker-compose-with-hdfs.yml @@ -2,7 +2,7 @@ version: "3" services: compactor-0: - image: ghcr.io/risingwavelabs/risingwave:RisingWave_v1.5.4_HDFS_2.7-x86_64 + image: ghcr.io/risingwavelabs/risingwave:RisingWave_1.6.1_HDFS_2.7-x86_64 command: - compactor-node - "--listen-addr" @@ -42,7 +42,7 @@ services: reservations: memory: 1G compute-node-0: - image: "ghcr.io/risingwavelabs/risingwave:RisingWave_v1.5.4_HDFS_2.7-x86_64" + image: "ghcr.io/risingwavelabs/risingwave:RisingWave_1.6.1_HDFS_2.7-x86_64" command: - compute-node - "--listen-addr" @@ -132,7 +132,7 @@ services: retries: 5 restart: always frontend-node-0: - image: "ghcr.io/risingwavelabs/risingwave:RisingWave_v1.5.4_HDFS_2.7-x86_64" + image: "ghcr.io/risingwavelabs/risingwave:RisingWave_1.6.1_HDFS_2.7-x86_64" command: - frontend-node - "--listen-addr" @@ -195,7 +195,7 @@ services: retries: 5 restart: always meta-node-0: - image: "ghcr.io/risingwavelabs/risingwave:RisingWave_v1.5.4_HDFS_2.7-x86_64" + image: "ghcr.io/risingwavelabs/risingwave:RisingWave_1.6.1_HDFS_2.7-x86_64" command: - meta-node - "--listen-addr" @@ -206,6 +206,8 @@ services: - "0.0.0.0:5691" - "--prometheus-host" - "0.0.0.0:1250" + - "--prometheus-endpoint" + - "http://prometheus-0:9500" - "--backend" - etcd - "--etcd-endpoints" diff --git a/docker/docker-compose-with-local-fs.yml b/docker/docker-compose-with-local-fs.yml index 9ba3b3d7326cd..8ffc0992747a3 100644 --- a/docker/docker-compose-with-local-fs.yml +++ b/docker/docker-compose-with-local-fs.yml @@ -10,6 +10,7 @@ services: --advertise-addr 0.0.0.0:5690 \ --dashboard-host 0.0.0.0:5691 \ --prometheus-host 0.0.0.0:1250 \ + --prometheus-endpoint http://prometheus-0:9500 \ --connector-rpc-endpoint 0.0.0.0:50051 \ --backend etcd \ --etcd-endpoints etcd-0:2388 \ @@ -19,7 +20,7 @@ services: --compute-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:5688 \ - --prometheus-listener-addr 0.0.0.0:1222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ --connector-rpc-endpoint 0.0.0.0:50051 \ @@ -30,32 +31,26 @@ services: --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ - --prometheus-listener-addr 0.0.0.0:2222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ --meta-addr http://0.0.0.0:5690\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ - --prometheus-listener-addr 0.0.0.0:1260 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ --meta-address http://0.0.0.0:5690\"" expose: - "6660" - - "1260" - "4566" - "5688" - - "1222" - "5690" - "1250" - "5691" - - "2222" ports: - "4566:4566" - "5690:5690" - "5691:5691" - - "1222:1222" - "1250:1250" - - "1260:1260" - - "2222:2222" depends_on: - etcd-0 volumes: diff --git a/docker/docker-compose-with-obs.yml b/docker/docker-compose-with-obs.yml index c7c397c8b1234..29d1c1a7452b9 100644 --- a/docker/docker-compose-with-obs.yml +++ b/docker/docker-compose-with-obs.yml @@ -1,7 +1,7 @@ --- version: "3" x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.0} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.1} services: risingwave-standalone: <<: *image @@ -10,6 +10,7 @@ services: --advertise-addr 0.0.0.0:5690 \ --dashboard-host 0.0.0.0:5691 \ --prometheus-host 0.0.0.0:1250 \ + --prometheus-endpoint http://prometheus-0:9500 \ --connector-rpc-endpoint 0.0.0.0:50051 \ --backend etcd \ --etcd-endpoints etcd-0:2388 \ @@ -19,7 +20,7 @@ services: --compute-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:5688 \ - --prometheus-listener-addr 0.0.0.0:1222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ --connector-rpc-endpoint 0.0.0.0:50051 \ @@ -31,32 +32,26 @@ services: --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ - --prometheus-listener-addr 0.0.0.0:2222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ --meta-addr http://0.0.0.0:5690\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ - --prometheus-listener-addr 0.0.0.0:1260 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ --meta-address http://0.0.0.0:5690\"" expose: - "6660" - - "1260" - "4566" - "5688" - - "1222" - "5690" - "1250" - "5691" - - "2222" ports: - "4566:4566" - "5690:5690" - "5691:5691" - - "1222:1222" - "1250:1250" - - "1260:1260" - - "2222:2222" depends_on: - etcd-0 env_file: multiple_object_storage.env diff --git a/docker/docker-compose-with-oss.yml b/docker/docker-compose-with-oss.yml index fc05c05dec207..b759d16a93d24 100644 --- a/docker/docker-compose-with-oss.yml +++ b/docker/docker-compose-with-oss.yml @@ -1,7 +1,7 @@ --- version: "3" x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.0} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.1} services: risingwave-standalone: <<: *image @@ -10,6 +10,7 @@ services: --advertise-addr 0.0.0.0:5690 \ --dashboard-host 0.0.0.0:5691 \ --prometheus-host 0.0.0.0:1250 \ + --prometheus-endpoint http://prometheus-0:9500 \ --connector-rpc-endpoint 0.0.0.0:50051 \ --backend etcd \ --etcd-endpoints etcd-0:2388 \ @@ -19,7 +20,7 @@ services: --compute-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:5688 \ - --prometheus-listener-addr 0.0.0.0:1222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ --connector-rpc-endpoint 0.0.0.0:50051 \ @@ -31,32 +32,26 @@ services: --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ - --prometheus-listener-addr 0.0.0.0:2222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ --meta-addr http://0.0.0.0:5690\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ - --prometheus-listener-addr 0.0.0.0:1260 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ --meta-address http://0.0.0.0:5690\"" expose: - "6660" - - "1260" - "4566" - "5688" - - "1222" - "5690" - "1250" - "5691" - - "2222" ports: - "4566:4566" - "5690:5690" - "5691:5691" - - "1222:1222" - "1250:1250" - - "1260:1260" - - "2222:2222" depends_on: - etcd-0 env_file: multiple_object_storage.env diff --git a/docker/docker-compose-with-s3.yml b/docker/docker-compose-with-s3.yml index e62955455bd88..a3070dd8048d2 100644 --- a/docker/docker-compose-with-s3.yml +++ b/docker/docker-compose-with-s3.yml @@ -1,7 +1,7 @@ --- version: "3" x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.0} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.1} services: risingwave-standalone: <<: *image @@ -10,6 +10,7 @@ services: --advertise-addr 0.0.0.0:5690 \ --dashboard-host 0.0.0.0:5691 \ --prometheus-host 0.0.0.0:1250 \ + --prometheus-endpoint http://prometheus-0:9500 \ --connector-rpc-endpoint 0.0.0.0:50051 \ --backend etcd \ --etcd-endpoints etcd-0:2388 \ @@ -19,7 +20,7 @@ services: --compute-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:5688 \ - --prometheus-listener-addr 0.0.0.0:1222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ --connector-rpc-endpoint 0.0.0.0:50051 \ @@ -31,32 +32,26 @@ services: --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ - --prometheus-listener-addr 0.0.0.0:2222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ --meta-addr http://0.0.0.0:5690\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ - --prometheus-listener-addr 0.0.0.0:1260 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ --meta-address http://0.0.0.0:5690\"" expose: - "6660" - - "1260" - "4566" - "5688" - - "1222" - "5690" - "1250" - "5691" - - "2222" ports: - "4566:4566" - "5690:5690" - "5691:5691" - - "1222:1222" - "1250:1250" - - "1260:1260" - - "2222:2222" depends_on: - etcd-0 env_file: aws.env diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index f7beb587e387e..a9d4cc0f58f7b 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -1,7 +1,7 @@ --- version: "3" x-image: &image - image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.0} + image: ${RW_IMAGE:-risingwavelabs/risingwave:v1.6.1} services: risingwave-standalone: <<: *image @@ -10,6 +10,7 @@ services: --advertise-addr 0.0.0.0:5690 \ --dashboard-host 0.0.0.0:5691 \ --prometheus-host 0.0.0.0:1250 \ + --prometheus-endpoint http://prometheus-0:9500 \ --connector-rpc-endpoint 0.0.0.0:50051 \ --backend etcd \ --etcd-endpoints etcd-0:2388 \ @@ -19,7 +20,7 @@ services: --compute-opts=\" \ --config-path /risingwave.toml \ --listen-addr 0.0.0.0:5688 \ - --prometheus-listener-addr 0.0.0.0:1222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:5688 \ --async-stack-trace verbose \ --connector-rpc-endpoint 0.0.0.0:50051 \ @@ -31,32 +32,26 @@ services: --config-path /risingwave.toml \ --listen-addr 0.0.0.0:4566 \ --advertise-addr 0.0.0.0:4566 \ - --prometheus-listener-addr 0.0.0.0:2222 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --health-check-listener-addr 0.0.0.0:6786 \ --meta-addr http://0.0.0.0:5690\" \ --compactor-opts=\" \ --listen-addr 0.0.0.0:6660 \ - --prometheus-listener-addr 0.0.0.0:1260 \ + --prometheus-listener-addr 0.0.0.0:1250 \ --advertise-addr 0.0.0.0:6660 \ --meta-address http://0.0.0.0:5690\"" expose: - "6660" - - "1260" - "4566" - "5688" - - "1222" - "5690" - "1250" - "5691" - - "2222" ports: - "4566:4566" - "5690:5690" - "5691:5691" - - "1222:1222" - "1250:1250" - - "1260:1260" - - "2222:2222" depends_on: - etcd-0 - minio-0 diff --git a/docker/prometheus.yaml b/docker/prometheus.yaml index f31a75e089d3a..d130a9e53f3be 100644 --- a/docker/prometheus.yaml +++ b/docker/prometheus.yaml @@ -37,18 +37,6 @@ scrape_configs: static_configs: - targets: ["message_queue:9644"] - - job_name: standalone-compute + - job_name: standalone static_configs: - - targets: ["risingwave-standalone:1222"] - - - job_name: standalone-meta - static_configs: - - targets: ["risingwave-standalone:1250"] - - - job_name: standalone-compactor - static_configs: - - targets: ["risingwave-standalone:1260"] - - - job_name: standalone-frontend - static_configs: - - targets: ["risingwave-standalone:2222"] + - targets: ["risingwave-standalone:1250"] \ No newline at end of file diff --git a/docs/developer-guide.md b/docs/developer-guide.md index 81e9aae0048bb..8cf5dcd4ebbaa 100644 --- a/docs/developer-guide.md +++ b/docs/developer-guide.md @@ -75,6 +75,7 @@ RiseDev is the development mode of RisingWave. To develop RisingWave, you need t * PostgreSQL (psql) (>= 14.1) * Tmux (>= v3.2a) * LLVM 16 (For macOS only, to workaround some bugs in macOS toolchain. See https://github.com/risingwavelabs/risingwave/issues/6205) +* Python (>= 3.12) (Optional, only required by `embedded-python-udf` feature) To install the dependencies on macOS, run: @@ -96,6 +97,31 @@ Then you'll be able to compile and start RiseDev! > > `.cargo/config.toml` contains `rustflags` configurations like `-Clink-arg` and `-Ctarget-feature`. Since it will be [merged](https://doc.rust-lang.org/cargo/reference/config.html#hierarchical-structure) with `$HOME/.cargo/config.toml`, check the config files and make sure they don't conflict if you have global `rustflags` configurations for e.g. linker there. +> [!INFO] +> +> If you want to build RisingWave with `embedded-python-udf` feature, you need to install Python 3.12. +> +> To install Python 3.12 on macOS, run: +> +> ```shell +> brew install python@3.12 +> ``` +> +> To install Python 3.12 on Debian-based Linux systems, run: +> +> ```shell +> sudo apt install software-properties-common +> sudo add-apt-repository ppa:deadsnakes/ppa +> sudo apt-get update +> sudo apt-get install python3.12 python3.12-dev +> ``` +> +> If the default `python3` version is not 3.12, please set the `PYO3_PYTHON` environment variable: +> +> ```shell +> export PYO3_PYTHON=python3.12 +> ``` + ## Start and monitor a dev cluster You can now build RiseDev and start a dev cluster. It is as simple as: diff --git a/docs/rustdoc/index.md b/docs/rustdoc/index.md index d82b1899c79a9..d8fe56471edb8 100644 --- a/docs/rustdoc/index.md +++ b/docs/rustdoc/index.md @@ -30,9 +30,9 @@ There's also a [Dash](https://kapeli.com/dash) docset feed containing all crate - [risingwave_hummock_sdk](risingwave_hummock_sdk/index.html) - [risingwave_object_store](risingwave_object_store/index.html) -### Source and Connector +### DML and Connectors (Source and Sink) -- [risingwave_source](risingwave_source/index.html) +- [risingwave_dml](risingwave_dml/index.html) - [risingwave_connector](risingwave_connector/index.html) ### Common diff --git a/e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt b/e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt new file mode 100644 index 0000000000000..a5a84ee3157c6 --- /dev/null +++ b/e2e_test/backfill/runtime/create_arrangement_backfill_mv.slt @@ -0,0 +1,5 @@ +statement ok +SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=true; + +statement ok +CREATE MATERIALIZED VIEW arrangement_backfill AS SELECT * FROM t; \ No newline at end of file diff --git a/e2e_test/backfill/runtime/create_no_shuffle_mv.slt b/e2e_test/backfill/runtime/create_no_shuffle_mv.slt new file mode 100644 index 0000000000000..4d867be59e8c3 --- /dev/null +++ b/e2e_test/backfill/runtime/create_no_shuffle_mv.slt @@ -0,0 +1,5 @@ +statement ok +SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=false; + +statement ok +CREATE MATERIALIZED VIEW no_shuffle_backfill AS SELECT * FROM t; \ No newline at end of file diff --git a/e2e_test/backfill/runtime/create_table.slt b/e2e_test/backfill/runtime/create_table.slt new file mode 100644 index 0000000000000..9f1c82a0ee087 --- /dev/null +++ b/e2e_test/backfill/runtime/create_table.slt @@ -0,0 +1,2 @@ +statement ok +CREATE TABLE t (v1 int primary key, v2 varchar, v3 bigint); \ No newline at end of file diff --git a/e2e_test/backfill/runtime/create_wide_table.slt b/e2e_test/backfill/runtime/create_wide_table.slt new file mode 100644 index 0000000000000..43453ebb5d2db --- /dev/null +++ b/e2e_test/backfill/runtime/create_wide_table.slt @@ -0,0 +1,20 @@ +statement ok +CREATE TABLE t ( + v1 int, + v2 varchar, + v3 bigint, + v4 int, + v5 varchar, + v6 bigint, + v7 int, + v8 varchar, + v9 bigint, + v10 int, + v11 varchar, + v12 bigint, + v13 int, + v14 varchar, + v15 bigint, + primary key (v1, v2, v3, v4, v5, + v8, v9, v10, v11, v12) +); \ No newline at end of file diff --git a/e2e_test/backfill/runtime/insert_snapshot.slt b/e2e_test/backfill/runtime/insert_snapshot.slt new file mode 100644 index 0000000000000..30c04c65d29af --- /dev/null +++ b/e2e_test/backfill/runtime/insert_snapshot.slt @@ -0,0 +1,5 @@ +statement ok +INSERT INTO t select generate_series, 'jakbj2khbe2', 22222222222 from generate_series(4000001, 8000000); + +statement ok +flush; \ No newline at end of file diff --git a/e2e_test/backfill/runtime/insert_upstream.slt b/e2e_test/backfill/runtime/insert_upstream.slt new file mode 100644 index 0000000000000..3aefbb35f6920 --- /dev/null +++ b/e2e_test/backfill/runtime/insert_upstream.slt @@ -0,0 +1,5 @@ +statement ok +INSERT INTO t select generate_series, 'jakbj2khbe2', 22222222222 from generate_series(1, 4000000); + +statement ok +flush; \ No newline at end of file diff --git a/e2e_test/backfill/runtime/insert_wide_snapshot.slt b/e2e_test/backfill/runtime/insert_wide_snapshot.slt new file mode 100644 index 0000000000000..cd4d2bfadeb6f --- /dev/null +++ b/e2e_test/backfill/runtime/insert_wide_snapshot.slt @@ -0,0 +1,22 @@ +# 15 columns wide +statement ok +INSERT INTO t select + generate_series, + 'jakbj2khbe2', + 22222222222, + generate_series, + 'jakbj2khbe2', + 22222222222, + generate_series, + 'jakbj2khbe2', + 22222222222, + generate_series, + 'jakbj2khbe2', + 22222222222, + generate_series, + 'jakbj2khbe2', + 22222222222 + from generate_series(2000001, 4000000); + +statement ok +flush; \ No newline at end of file diff --git a/e2e_test/backfill/runtime/validate_rows_arrangement.slt b/e2e_test/backfill/runtime/validate_rows_arrangement.slt new file mode 100644 index 0000000000000..31f5041c1ea61 --- /dev/null +++ b/e2e_test/backfill/runtime/validate_rows_arrangement.slt @@ -0,0 +1,4 @@ +query I +select (select count(*) from arrangement_backfill) = (select count(*) from t); +---- +t \ No newline at end of file diff --git a/e2e_test/backfill/runtime/validate_rows_no_shuffle.slt b/e2e_test/backfill/runtime/validate_rows_no_shuffle.slt new file mode 100644 index 0000000000000..23a616de78448 --- /dev/null +++ b/e2e_test/backfill/runtime/validate_rows_no_shuffle.slt @@ -0,0 +1,4 @@ +query I +select (select count(*) from no_shuffle_backfill) = (select count(*) from t); +---- +t \ No newline at end of file diff --git a/e2e_test/batch/basic/case_when_optimization.slt.part b/e2e_test/batch/basic/case_when_optimization.slt.part new file mode 100644 index 0000000000000..7e01be030a911 --- /dev/null +++ b/e2e_test/batch/basic/case_when_optimization.slt.part @@ -0,0 +1,242 @@ +statement ok +SET RW_IMPLICIT_FLUSH TO true; + +statement ok +CREATE TABLE t1 (c1 INT, c2 INT, c3 INT); + +statement ok +INSERT INTO t1 VALUES (1, 1, 1), (2, 2, 2), (3, 3, 3), (4, 4, 4), (5, 5, 5), + (6, 6, 6), (7, 7, 7), (8, 8, 8), (9, 9, 9), (10, 10, 10), + (11, 11, 11), (12, 12, 12), (13, 13, 13), (14, 14, 14), (15, 15, 15), + (16, 16, 16), (17, 17, 17), (18, 18, 18), (19, 19, 19), (20, 20, 20), + (21, 21, 21), (22, 22, 22), (23, 23, 23), (24, 24, 24), (25, 25, 25), + (26, 26, 26), (27, 27, 27), (28, 28, 28), (29, 29, 29), (30, 30, 30), + (31, 31, 31), (32, 32, 32), (33, 33, 33), (34, 34, 34), (35, 35, 35), + (36, 36, 36), (37, 37, 37), (38, 38, 38), (39, 39, 39), (40, 40, 40), + (41, 41, 41), (42, 42, 42), (43, 43, 43), (44, 44, 44), (45, 45, 45), + (46, 46, 46), (47, 47, 47), (48, 48, 48), (49, 49, 49), (50, 50, 50), + (51, 51, 51), (52, 52, 52), (53, 53, 53), (54, 54, 54), (55, 55, 55), + (56, 56, 56), (57, 57, 57), (58, 58, 58), (59, 59, 59), (60, 60, 60), + (61, 61, 61), (62, 62, 62), (63, 63, 63), (64, 64, 64), (65, 65, 65), + (66, 66, 66), (67, 67, 67), (68, 68, 68), (69, 69, 69), (70, 70, 70), + (71, 71, 71), (72, 72, 72), (73, 73, 73), (74, 74, 74), (75, 75, 75), + (76, 76, 76), (77, 77, 77), (78, 78, 78), (79, 79, 79), (80, 80, 80), + (81, 81, 81), (82, 82, 82), (83, 83, 83), (84, 84, 84), (85, 85, 85), + (86, 86, 86), (87, 87, 87), (88, 88, 88), (89, 89, 89), (90, 90, 90), + (91, 91, 91), (92, 92, 92), (93, 93, 93), (94, 94, 94), (95, 95, 95), + (96, 96, 96), (97, 97, 97), (98, 98, 98), (99, 99, 99), (100, 100, 100); + + +# 101 arms case-when expression, with optimizable pattern +query I +SELECT + CASE c1 + WHEN 1 THEN 'one' + WHEN 2 THEN 'two' + WHEN 3 THEN 'three' + WHEN 4 THEN 'four' + WHEN 5 THEN 'five' + WHEN 6 THEN 'six' + WHEN 7 THEN 'seven' + WHEN 8 THEN 'eight' + WHEN 9 THEN 'nine' + WHEN 10 THEN 'ten' + WHEN 11 THEN 'eleven' + WHEN 12 THEN 'twelve' + WHEN 13 THEN 'thirteen' + WHEN 14 THEN 'fourteen' + WHEN 15 THEN 'fifteen' + WHEN 16 THEN 'sixteen' + WHEN 17 THEN 'seventeen' + WHEN 18 THEN 'eighteen' + WHEN 19 THEN 'nineteen' + WHEN 20 THEN 'twenty' + WHEN 21 THEN 'twenty-one' + WHEN 22 THEN 'twenty-two' + WHEN 23 THEN 'twenty-three' + WHEN 24 THEN 'twenty-four' + WHEN 25 THEN 'twenty-five' + WHEN 26 THEN 'twenty-six' + WHEN 27 THEN 'twenty-seven' + WHEN 28 THEN 'twenty-eight' + WHEN 29 THEN 'twenty-nine' + WHEN 30 THEN 'thirty' + WHEN 31 THEN 'thirty-one' + WHEN 32 THEN 'thirty-two' + WHEN 33 THEN 'thirty-three' + WHEN 34 THEN 'thirty-four' + WHEN 35 THEN 'thirty-five' + WHEN 36 THEN 'thirty-six' + WHEN 37 THEN 'thirty-seven' + WHEN 38 THEN 'thirty-eight' + WHEN 39 THEN 'thirty-nine' + WHEN 40 THEN 'forty' + WHEN 41 THEN 'forty-one' + WHEN 42 THEN 'forty-two' + WHEN 43 THEN 'forty-three' + WHEN 44 THEN 'forty-four' + WHEN 45 THEN 'forty-five' + WHEN 46 THEN 'forty-six' + WHEN 47 THEN 'forty-seven' + WHEN 48 THEN 'forty-eight' + WHEN 49 THEN 'forty-nine' + WHEN 50 THEN 'fifty' + WHEN 51 THEN 'fifty-one' + WHEN 52 THEN 'fifty-two' + WHEN 53 THEN 'fifty-three' + WHEN 54 THEN 'fifty-four' + WHEN 55 THEN 'fifty-five' + WHEN 56 THEN 'fifty-six' + WHEN 57 THEN 'fifty-seven' + WHEN 58 THEN 'fifty-eight' + WHEN 59 THEN 'fifty-nine' + WHEN 60 THEN 'sixty' + WHEN 61 THEN 'sixty-one' + WHEN 62 THEN 'sixty-two' + WHEN 63 THEN 'sixty-three' + WHEN 64 THEN 'sixty-four' + WHEN 65 THEN 'sixty-five' + WHEN 66 THEN 'sixty-six' + WHEN 67 THEN 'sixty-seven' + WHEN 68 THEN 'sixty-eight' + WHEN 69 THEN 'sixty-nine' + WHEN 70 THEN 'seventy' + WHEN 71 THEN 'seventy-one' + WHEN 72 THEN 'seventy-two' + WHEN 73 THEN 'seventy-three' + WHEN 74 THEN 'seventy-four' + WHEN 75 THEN 'seventy-five' + WHEN 76 THEN 'seventy-six' + WHEN 77 THEN 'seventy-seven' + WHEN 78 THEN 'seventy-eight' + WHEN 79 THEN 'seventy-nine' + WHEN 80 THEN 'eighty' + WHEN 81 THEN 'eighty-one' + WHEN 82 THEN 'eighty-two' + WHEN 83 THEN 'eighty-three' + WHEN 84 THEN 'eighty-four' + WHEN 85 THEN 'eighty-five' + WHEN 86 THEN 'eighty-six' + WHEN 87 THEN 'eighty-seven' + WHEN 88 THEN 'eighty-eight' + WHEN 89 THEN 'eighty-nine' + WHEN 90 THEN 'ninety' + WHEN 91 THEN 'ninety-one' + WHEN 92 THEN 'ninety-two' + WHEN 93 THEN 'ninety-three' + WHEN 94 THEN 'ninety-four' + WHEN 95 THEN 'ninety-five' + WHEN 96 THEN 'ninety-six' + WHEN 97 THEN 'ninety-seven' + WHEN 98 THEN 'ninety-eight' + WHEN 99 THEN 'ninety-nine' + WHEN 100 THEN 'one hundred' + ELSE + '114514' + END +FROM t1 +ORDER BY c1 ASC; +---- +one +two +three +four +five +six +seven +eight +nine +ten +eleven +twelve +thirteen +fourteen +fifteen +sixteen +seventeen +eighteen +nineteen +twenty +twenty-one +twenty-two +twenty-three +twenty-four +twenty-five +twenty-six +twenty-seven +twenty-eight +twenty-nine +thirty +thirty-one +thirty-two +thirty-three +thirty-four +thirty-five +thirty-six +thirty-seven +thirty-eight +thirty-nine +forty +forty-one +forty-two +forty-three +forty-four +forty-five +forty-six +forty-seven +forty-eight +forty-nine +fifty +fifty-one +fifty-two +fifty-three +fifty-four +fifty-five +fifty-six +fifty-seven +fifty-eight +fifty-nine +sixty +sixty-one +sixty-two +sixty-three +sixty-four +sixty-five +sixty-six +sixty-seven +sixty-eight +sixty-nine +seventy +seventy-one +seventy-two +seventy-three +seventy-four +seventy-five +seventy-six +seventy-seven +seventy-eight +seventy-nine +eighty +eighty-one +eighty-two +eighty-three +eighty-four +eighty-five +eighty-six +eighty-seven +eighty-eight +eighty-nine +ninety +ninety-one +ninety-two +ninety-three +ninety-four +ninety-five +ninety-six +ninety-seven +ninety-eight +ninety-nine +one hundred + +statement ok +drop table t1; \ No newline at end of file diff --git a/e2e_test/batch/basic/logical_view.slt.part b/e2e_test/batch/basic/logical_view.slt.part index 4adc05b386067..9c6b4c0f7360d 100644 --- a/e2e_test/batch/basic/logical_view.slt.part +++ b/e2e_test/batch/basic/logical_view.slt.part @@ -40,7 +40,7 @@ SELECT * FROM v3; statement error DROP TABLE t; -statement error other relation\(s\) depend on it +statement error Permission denied DROP VIEW v2; statement ok diff --git a/e2e_test/batch/basic/make_time.slt.part b/e2e_test/batch/basic/make_time.slt.part new file mode 100644 index 0000000000000..ff1d4453e0efd --- /dev/null +++ b/e2e_test/batch/basic/make_time.slt.part @@ -0,0 +1,185 @@ +statement ok +SET RW_IMPLICIT_FLUSH TO true; + +query T +SELECT make_timestamptz(1973, 07, 15, 08, 15, 55.33); +---- +1973-07-15 08:15:55.330+00:00 + +query T +SELECT make_timestamptz(-1973, 07, 15, 08, 15, 55.33); +---- +1973-07-15 08:15:55.330+00:00 BC + +query T +SELECT make_timestamptz(20240, 1, 26, 14, 20, 26); +---- +20240-01-26 14:20:26+00:00 + +query error Invalid parameter year, month, day: invalid date: -3-2-29 +SELECT make_timestamptz(-4, 02, 29, 08, 15, 55.33); + +query T +SELECT make_timestamptz(-5, 02, 29, 08, 15, 55.33); +---- +0005-02-29 08:15:55.330+00:00 BC + +query error Invalid parameter sec: invalid sec: -55.33 +SELECT make_timestamptz(1973, 07, 15, 08, 15, -55.33); + +query error Invalid parameter hour, min, sec: invalid time: 8:-15:55.33 +SELECT make_timestamptz(1973, 07, 15, 08, -15, 55.33); + +query error Invalid parameter year, month, day: invalid date: 1973--7-15 +SELECT make_timestamptz(1973, -07, 15, 08, 15, 55.33); + +query error Invalid parameter year, month, day: invalid date: 1973-6-31 +SELECT make_timestamptz(1973, 06, 31, 08, 15, 55.33); + +query error Invalid parameter year, month, day: invalid date: 0-6-31 +SELECT make_timestamptz(0, 06, 31, 08, 15, 55.33); + +statement ok +set TimeZone to 'America/New_York'; + +query T +SELECT make_timestamptz(1973, 07, 15, 08, 15, 55.33); +---- +1973-07-15 08:15:55.330-04:00 + +statement ok +create table ttz(tstz timestamptz); + +statement ok +insert into ttz values(make_timestamptz(1973, 06, 25, 08, 15, 55.33)); + +query TT +select * from ttz; +---- +1973-06-25 08:15:55.330-04:00 + +statement ok +drop table ttz; + +query error Invalid parameter time_zone: 'Nehwon/Lankhmar' is not a valid timezone +SELECT make_timestamptz(1910, 12, 24, 0, 0, 0, 'Nehwon/Lankhmar'); + +query TT +WITH tzs (tz) AS (VALUES ('Europe/Prague'), ('Europe/Paris'), ('America/New_York'), ('EST'), ('EST5EDT'), ('PST8PDT')) SELECT make_timestamptz(2010, 2, 27, 3, 45, 00, tz), tz FROM tzs; +---- +2010-02-26 21:45:00-05:00 Europe/Prague +2010-02-26 21:45:00-05:00 Europe/Paris +2010-02-27 03:45:00-05:00 America/New_York +2010-02-27 03:45:00-05:00 EST +2010-02-27 03:45:00-05:00 EST5EDT +2010-02-27 06:45:00-05:00 PST8PDT + +query TT +WITH tzs (tz) AS (VALUES ('Europe/Prague'), ('Europe/Paris'), ('America/New_York'), ('EST'), ('EST5EDT'), ('PST8PDT')) SELECT make_timestamptz(2010, 2, 27, 3, 45, 00, tz) AT TIME ZONE 'EST5EDT', tz FROM tzs; +---- +2010-02-26 21:45:00 Europe/Prague +2010-02-26 21:45:00 Europe/Paris +2010-02-27 03:45:00 America/New_York +2010-02-27 03:45:00 EST +2010-02-27 03:45:00 EST5EDT +2010-02-27 06:45:00 PST8PDT + +query T +SELECT make_timestamptz(1973, 07, 15, 08, 15, 55.33, 'Asia/Manila') = '1973-07-14 20:15:55.33'::timestamptz; +---- +t + +statement ok +set TimeZone to 'Europe/London'; + +query T +SELECT make_timestamptz(2013, 7, 15, 8, 15, 23.5); +---- +2013-07-15 08:15:23.500+01:00 + +query T +SELECT make_timestamptz(2013, 7, 15, 8, 15, 23.5, 'America/New_York'); +---- +2013-07-15 13:15:23.500+01:00 + +statement ok +set timezone to 'UTC'; + +query T +SELECT make_date(2024, 1, 26); +---- +2024-01-26 + +query T +SELECT make_date(20240, 1, 26); +---- +20240-01-26 + +query T +SELECT make_date(-2024, 1, 26); +---- +2024-01-26 BC + +query error Invalid parameter year, month, day: invalid date: -3-2-29 +SELECT make_date(-4, 2, 29); + +query T +SELECT make_date(-5, 2, 29); +---- +0005-02-29 BC + +query error Invalid parameter year, month, day: invalid date: 0-7-15 +select make_date(0, 7, 15); + +query error Invalid parameter year, month, day: invalid date: 2013-2-30 +select make_date(2013, 2, 30); + +query error Invalid parameter year, month, day: invalid date: 2013-13-1 +select make_date(2013, 13, 1); + +query error Invalid parameter year, month, day: invalid date: 2013-11--1 +select make_date(2013, 11, -1); + +query error Invalid parameter hour, min, sec: invalid time: 10:55:100.1 +select make_time(10, 55, 100.1); + +query T +SELECT make_time(14, 20, 26); +---- +14:20:26 + +query error Invalid parameter hour, min, sec: invalid time: 24:0:2.1 +select make_time(24, 0, 2.1); + +query T +SELECT make_timestamp(2024, 1, 26, 14, 20, 26); +---- +2024-01-26 14:20:26 + +query T +SELECT make_timestamp(20240, 1, 26, 14, 20, 26); +---- +20240-01-26 14:20:26 + +query T +SELECT make_timestamp(-1973, 07, 15, 08, 15, 55.33); +---- +1973-07-15 08:15:55.330 BC + +query error Invalid parameter year, month, day: invalid date: -3-2-29 +SELECT make_timestamp(-4, 02, 29, 08, 15, 55.33); + +query T +SELECT make_timestamp(-5, 02, 29, 08, 15, 55.33); +---- +0005-02-29 08:15:55.330 BC + +query T +select '0001-01-01 12:34:56'::timestamp - '10 year'::interval; +---- +0010-01-01 12:34:56 BC + +query T +select '0001-01-01 12:34:56'::timestamptz - '10 year'::interval; +---- +0010-01-01 12:34:56+00:00 BC diff --git a/e2e_test/batch/basic/make_timestamptz.slt.part b/e2e_test/batch/basic/make_timestamptz.slt.part deleted file mode 100644 index 99f5d1369327c..0000000000000 --- a/e2e_test/batch/basic/make_timestamptz.slt.part +++ /dev/null @@ -1,90 +0,0 @@ -statement ok -SET RW_IMPLICIT_FLUSH TO true; - -query T -SELECT make_timestamptz(1973, 07, 15, 08, 15, 55.33); ----- -1973-07-15 08:15:55.330+00:00 - -query T -SELECT make_timestamptz(-1973, 07, 15, 08, 15, 55.33); ----- --1973-07-15 08:15:55.330+00:00 - -query error Invalid parameter sec: invalid sec -SELECT make_timestamptz(1973, 07, 15, 08, 15, -55.33); - -query error Invalid parameter hour, min, sec: invalid time -SELECT make_timestamptz(1973, 07, 15, 08, -15, 55.33); - -query error Invalid parameter year, month, day: invalid date -SELECT make_timestamptz(1973, -07, 15, 08, 15, 55.33); - -query error Invalid parameter year, month, day: invalid date -SELECT make_timestamptz(1973, 06, 31, 08, 15, 55.33); - -statement ok -set TimeZone to 'America/New_York'; - -query T -SELECT make_timestamptz(1973, 07, 15, 08, 15, 55.33); ----- -1973-07-15 08:15:55.330-04:00 - -statement ok -create table ttz(tstz timestamptz); - -statement ok -insert into ttz values(make_timestamptz(1973, 06, 25, 08, 15, 55.33)); - -query TT -select * from ttz; ----- -1973-06-25 08:15:55.330-04:00 - -statement ok -drop table ttz; - -query error Invalid parameter time_zone: 'Nehwon/Lankhmar' is not a valid timezone -SELECT make_timestamptz(1910, 12, 24, 0, 0, 0, 'Nehwon/Lankhmar'); - -query TT -WITH tzs (tz) AS (VALUES ('Europe/Prague'), ('Europe/Paris'), ('America/New_York'), ('EST'), ('EST5EDT'), ('PST8PDT')) SELECT make_timestamptz(2010, 2, 27, 3, 45, 00, tz), tz FROM tzs; ----- -2010-02-26 21:45:00-05:00 Europe/Prague -2010-02-26 21:45:00-05:00 Europe/Paris -2010-02-27 03:45:00-05:00 America/New_York -2010-02-27 03:45:00-05:00 EST -2010-02-27 03:45:00-05:00 EST5EDT -2010-02-27 06:45:00-05:00 PST8PDT - -query TT -WITH tzs (tz) AS (VALUES ('Europe/Prague'), ('Europe/Paris'), ('America/New_York'), ('EST'), ('EST5EDT'), ('PST8PDT')) SELECT make_timestamptz(2010, 2, 27, 3, 45, 00, tz) AT TIME ZONE 'EST5EDT', tz FROM tzs; ----- -2010-02-26 21:45:00 Europe/Prague -2010-02-26 21:45:00 Europe/Paris -2010-02-27 03:45:00 America/New_York -2010-02-27 03:45:00 EST -2010-02-27 03:45:00 EST5EDT -2010-02-27 06:45:00 PST8PDT - -query T -SELECT make_timestamptz(1973, 07, 15, 08, 15, 55.33, 'Asia/Manila') = '1973-07-14 20:15:55.33'::timestamptz; ----- -t - -statement ok -set TimeZone to 'Europe/London'; - -query T -SELECT make_timestamptz(2013, 7, 15, 8, 15, 23.5); ----- -2013-07-15 08:15:23.500+01:00 - -query T -SELECT make_timestamptz(2013, 7, 15, 8, 15, 23.5, 'America/New_York'); ----- -2013-07-15 13:15:23.500+01:00 - -statement ok -set timezone to 'UTC'; diff --git a/e2e_test/batch/catalog/atlasgo.slt.part b/e2e_test/batch/catalog/atlasgo.slt.part new file mode 100644 index 0000000000000..a9b1671c2985a --- /dev/null +++ b/e2e_test/batch/catalog/atlasgo.slt.part @@ -0,0 +1,182 @@ +statement ok +CREATE TABLE IF NOT EXISTS data_types ( + id BIGINT PRIMARY KEY, + varchar_column VARCHAR, + text_column TEXT, + integer_column INTEGER, + smallint_column SMALLINT, + bigint_column BIGINT, + decimal_column DECIMAL, + real_column REAL, + double_column DOUBLE PRECISION, + boolean_column BOOLEAN, + date_column DATE, + time_column TIME, + timestamp_column TIMESTAMP, + timestamptz_column TIMESTAMPTZ, + jsonb_column JSONB, + bytea_column BYTEA +); + +query T +SELECT setting FROM pg_settings WHERE name IN ('server_version_num', 'crdb_version') ORDER BY name DESC; +---- +130014 + +query TT rowsort +SELECT + nspname AS schema_name, + pg_catalog.obj_description(oid) AS comment +FROM + pg_catalog.pg_namespace +WHERE + nspname NOT IN ('information_schema', 'pg_catalog', 'pg_toast', 'crdb_internal', 'pg_extension') + AND nspname NOT LIKE 'pg_%temp_%' +ORDER BY + nspname; +---- +public (empty) +rw_catalog (empty) + +query TTTTTTT +SELECT + t1.table_schema, + t1.table_name, + pg_catalog.obj_description(t3.oid, 'pg_class') AS comment, + t4.partattrs AS partition_attrs, + t4.partstrat AS partition_strategy, + pg_get_expr(t4.partexprs, t4.partrelid) AS partition_exprs +FROM + INFORMATION_SCHEMA.TABLES AS t1 + JOIN pg_catalog.pg_namespace AS t2 ON t2.nspname = t1.table_schema + JOIN pg_catalog.pg_class AS t3 ON t3.relnamespace = t2.oid AND t3.relname = t1.table_name + LEFT JOIN pg_catalog.pg_partitioned_table AS t4 ON t4.partrelid = t3.oid + LEFT JOIN pg_depend AS t5 ON t5.objid = t3.oid AND t5.deptype = 'e' +WHERE + t1.table_type = 'BASE TABLE' + AND NOT COALESCE(t3.relispartition, false) + AND t1.table_schema IN ('public', 'rw_catalog') + AND t5.objid IS NULL +ORDER BY + t1.table_schema, t1.table_name; +---- +public data_types (empty) NULL NULL (empty) + +query TTTT +SELECT + n.nspname AS schema_name, + e.enumtypid AS enum_id, + t.typname AS enum_name, + e.enumlabel AS enum_value +FROM + pg_enum e + JOIN pg_type t ON e.enumtypid = t.oid + JOIN pg_namespace n ON t.typnamespace = n.oid +WHERE + n.nspname IN ('public', 'rw_catalog') +ORDER BY + n.nspname, e.enumtypid, e.enumsortorder +---- + +query T rowsort +SELECT + ( + SELECT + t.typtype + FROM + pg_catalog.pg_type AS t + WHERE + t.oid = t4.typelem + ) AS elemtyp +FROM + pg_catalog.pg_type AS t4; +---- +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +NULL +b +b +b +b +b +b +b +b +b +b +b +b +b +b +b + +query TTTTTTTTTTTTTTTTTTTTTT +SELECT + t1.table_name, + t1.column_name, + t1.data_type, + pg_catalog.format_type(a.atttypid, a.atttypmod) AS format_type, + t1.is_nullable, + t1.column_default, + t1.character_maximum_length, + t1.numeric_precision, + t1.datetime_precision, + t1.numeric_scale, + t1.interval_type, + t1.character_set_name, + t1.collation_name, + t1.is_identity, + t1.identity_start, + t1.identity_increment, + -- (CASE WHEN t1.is_identity = 'YES' THEN (SELECT last_value FROM pg_sequences WHERE quote_ident(schemaname) || '.' || quote_ident(sequencename) = pg_get_serial_sequence(quote_ident(t1.table_schema) || '.' || quote_ident(t1.table_name), t1.column_name)) END) AS identity_last, + t1.identity_generation, + t1.generation_expression, + col_description(t3.oid, "ordinal_position") AS comment, + t4.typtype, + t4.typelem, + -- (CASE WHEN t4.typcategory = 'A' AND t4.typelem <> 0 THEN (SELECT t.typtype FROM pg_catalog.pg_type t WHERE t.oid = t4.typelem) END) AS elemtyp, + t4.oid +FROM + "information_schema"."columns" AS t1 + JOIN pg_catalog.pg_namespace AS t2 ON t2.nspname = t1.table_schema + JOIN pg_catalog.pg_class AS t3 ON t3.relnamespace = t2.oid AND t3.relname = t1.table_name + JOIN pg_catalog.pg_attribute AS a ON a.attrelid = t3.oid AND a.attname = t1.column_name + LEFT JOIN pg_catalog.pg_type AS t4 ON t4.oid = a.atttypid +WHERE + t1.table_schema = 'public' AND t1.table_name IN ('data_types') +ORDER BY + t1.table_name, t1.ordinal_position; +---- +data_types id bigint bigint YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 20 +data_types varchar_column character varying character varying YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 1043 +data_types text_column character varying character varying YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 1043 +data_types integer_column integer integer YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 23 +data_types smallint_column smallint smallint YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 21 +data_types bigint_column bigint bigint YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 20 +data_types decimal_column numeric numeric YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 1700 +data_types real_column real real YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 700 +data_types double_column double precision double precision YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 701 +data_types boolean_column boolean boolean YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 16 +data_types date_column date date YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 1082 +data_types time_column time without time zone time without time zone YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 1083 +data_types timestamp_column timestamp without time zone timestamp without time zone YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 1114 +data_types timestamptz_column timestamp with time zone timestamp with time zone YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 1184 +data_types jsonb_column jsonb jsonb YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 3802 +data_types bytea_column bytea bytea YES NULL NULL NULL NULL NULL NULL NULL NULL NO NULL NULL NULL NULL (empty) b 0 17 + +statement ok +DROP TABLE data_types; diff --git a/e2e_test/batch/catalog/issue_8791.slt.part b/e2e_test/batch/catalog/issue_8791.slt.part index b8339c44a23f4..482d0965ba543 100644 --- a/e2e_test/batch/catalog/issue_8791.slt.part +++ b/e2e_test/batch/catalog/issue_8791.slt.part @@ -1,16 +1,16 @@ # UNION and other complex queries should also be in local mode query I -SELECT name FROM pg_catalog.pg_settings union select 'a'; +SELECT amname FROM pg_catalog.pg_am union select 'a'; ---- a query T -SELECT name FROM (SELECT pg_catalog.lower(name) AS name FROM pg_catalog.pg_settings UNION ALL SELECT 'session authorization' UNION ALL SELECT 'all') ss WHERE substring(name,1,0)='' -LIMIT 1000 +SELECT amname FROM (SELECT pg_catalog.lower(amname) AS amname FROM pg_catalog.pg_am UNION ALL SELECT 'session authorization' UNION ALL SELECT 'all') ss WHERE substring(amname,1,0)='' +LIMIT 1000; ---- session authorization all query I -with q as ( select name FROM pg_catalog.pg_settings ) select * from q; +with q as ( select amname FROM pg_catalog.pg_am ) select * from q; ---- diff --git a/e2e_test/batch/catalog/pg_cast.slt.part b/e2e_test/batch/catalog/pg_cast.slt.part index b8ab68a5ed5cd..b1558d1e144c4 100644 --- a/e2e_test/batch/catalog/pg_cast.slt.part +++ b/e2e_test/batch/catalog/pg_cast.slt.part @@ -82,8 +82,9 @@ SELECT * FROM pg_catalog.pg_cast; 78 3802 701 e 79 3802 1700 e 80 3802 1043 a -81 1301 701 e -82 1301 1043 a +81 20 20 e +82 1301 701 e +83 1301 1043 a query TT rowsort SELECT s.typname, t.typname diff --git a/e2e_test/batch/catalog/pg_class.slt.part b/e2e_test/batch/catalog/pg_class.slt.part index 3b8a5162536e6..ff31c27dcc17d 100644 --- a/e2e_test/batch/catalog/pg_class.slt.part +++ b/e2e_test/batch/catalog/pg_class.slt.part @@ -1,23 +1,23 @@ query ITIT SELECT oid,relname,relowner,relkind FROM pg_catalog.pg_class ORDER BY oid limit 15; ---- -1 pg_type 1 v -2 pg_namespace 1 v -3 pg_cast 1 v -4 pg_matviews 1 v -5 pg_user 1 v -6 pg_class 1 v -7 pg_index 1 v -8 pg_opclass 1 v -9 pg_collation 1 v -10 pg_am 1 v -11 pg_operator 1 v -12 pg_views 1 v -13 pg_attribute 1 v -14 pg_database 1 v +1 columns 1 v +2 tables 1 v +3 views 1 v +4 pg_am 1 v +5 pg_attrdef 1 v +6 pg_attribute 1 v +7 pg_auth_members 1 v +8 pg_cast 1 r +9 pg_class 1 v +10 pg_collation 1 v +11 pg_constraint 1 r +12 pg_conversion 1 v +13 pg_database 1 v +14 pg_depend 1 v 15 pg_description 1 v query ITIT SELECT oid,relname,relowner,relkind FROM pg_catalog.pg_class WHERE oid = 'pg_namespace'::regclass; ---- -2 pg_namespace 1 v +25 pg_namespace 1 v diff --git a/e2e_test/batch/catalog/pg_constraint.slt.part b/e2e_test/batch/catalog/pg_constraint.slt.part new file mode 100644 index 0000000000000..a2a36e73f5416 --- /dev/null +++ b/e2e_test/batch/catalog/pg_constraint.slt.part @@ -0,0 +1,10 @@ +statement ok +create table t(a int, b int, c varchar, primary key(a,b)); + +query TTTT +select conname, contype, conkey from pg_constraint where conname='t_pkey'; +---- +t_pkey p {1,2} + +statement ok +drop table t; diff --git a/e2e_test/batch/catalog/pg_database.slt.part b/e2e_test/batch/catalog/pg_database.slt.part index 8b38ebac23e43..a2b610b678afd 100644 --- a/e2e_test/batch/catalog/pg_database.slt.part +++ b/e2e_test/batch/catalog/pg_database.slt.part @@ -1,4 +1,4 @@ query ITIITT -SELECT oid, datname, encoding, datcollate, datctype, datistemplate, datallowconn, datconnlimit, dattablespace FROM pg_catalog.pg_database where oid = 0; +SELECT datname, encoding, datcollate, datctype, datistemplate, datallowconn, datconnlimit, dattablespace FROM pg_catalog.pg_database where datname = 'dev'; ---- -0 dev 6 C C f t -1 1663 +dev 6 C C f t -1 1663 diff --git a/e2e_test/batch/catalog/pg_namespace.slt.part b/e2e_test/batch/catalog/pg_namespace.slt.part index 3ea9fd4141484..085f35a4eee4b 100644 --- a/e2e_test/batch/catalog/pg_namespace.slt.part +++ b/e2e_test/batch/catalog/pg_namespace.slt.part @@ -4,4 +4,4 @@ SELECT nspname FROM pg_catalog.pg_namespace; information_schema pg_catalog public -rw_catalog \ No newline at end of file +rw_catalog diff --git a/e2e_test/batch/catalog/pg_settings.slt.part b/e2e_test/batch/catalog/pg_settings.slt.part index 0481ab1a1dd5b..9ff41b8dbea45 100644 --- a/e2e_test/batch/catalog/pg_settings.slt.part +++ b/e2e_test/batch/catalog/pg_settings.slt.part @@ -1,7 +1,101 @@ query TT -SELECT * FROM pg_catalog.pg_settings; +SELECT context, name FROM pg_catalog.pg_settings ORDER BY (context, name); ---- +internal block_size_kb +internal bloom_false_positive +internal data_directory +internal parallel_compact_size_mb +internal sstable_size_mb +internal state_store +postmaster backup_storage_directory +postmaster backup_storage_url +postmaster barrier_interval_ms +postmaster checkpoint_frequency +postmaster enable_tracing +postmaster max_concurrent_creating_streaming_jobs +postmaster pause_on_next_bootstrap +user application_name +user background_ddl +user batch_enable_distributed_dml +user batch_parallelism +user bytea_output +user client_encoding +user client_min_messages +user create_compaction_group_for_mv +user datestyle +user extra_float_digits +user idle_in_transaction_session_timeout +user intervalstyle +user lock_timeout +user max_split_range_gap +user query_epoch +user query_mode +user row_security +user rw_batch_enable_lookup_join +user rw_batch_enable_sort_agg +user rw_enable_join_ordering +user rw_enable_share_plan +user rw_enable_two_phase_agg +user rw_force_split_distinct_agg +user rw_force_two_phase_agg +user rw_implicit_flush +user rw_streaming_allow_jsonb_in_stream_key +user rw_streaming_enable_bushy_join +user rw_streaming_enable_delta_join +user rw_streaming_over_window_cache_policy +user search_path +user server_encoding +user server_version +user server_version_num +user sink_decouple +user standard_conforming_strings +user statement_timeout +user streaming_enable_arrangement_backfill +user streaming_parallelism +user streaming_rate_limit +user synchronize_seqscans +user timezone +user transaction_isolation +user visibility_mode query TT SELECT * FROM pg_catalog.pg_settings where name='dummy'; ----- \ No newline at end of file +---- + +# https://github.com/risingwavelabs/risingwave/issues/15125 +query TT +SELECT min(name) name, context FROM pg_catalog.pg_settings GROUP BY context; +---- +application_name user +backup_storage_directory postmaster +block_size_kb internal + +# Tab-completion of `SET` command +query T +SELECT name +FROM + (SELECT pg_catalog.lower(name) AS name + FROM pg_catalog.pg_settings + WHERE context IN ('user', + 'superuser') + UNION ALL SELECT 'constraints' + UNION ALL SELECT 'transaction' + UNION ALL SELECT 'session' + UNION ALL SELECT 'role' + UNION ALL SELECT 'tablespace' + UNION ALL SELECT 'all') ss +WHERE substring(name, 1, 8)='search_p'; +---- +search_path + +# Tab-completion of `ALTER SYSTEM SET` command +query T +SELECT name +FROM + (SELECT pg_catalog.lower(name) AS name + FROM pg_catalog.pg_settings + WHERE context != 'internal' + UNION ALL SELECT 'all') ss +WHERE substring(name, 1, 7)='checkpo'; +---- +checkpoint_frequency diff --git a/e2e_test/batch/catalog/rw_depend.slt.part b/e2e_test/batch/catalog/rw_depend.slt.part new file mode 100644 index 0000000000000..96a3b5830846e --- /dev/null +++ b/e2e_test/batch/catalog/rw_depend.slt.part @@ -0,0 +1,43 @@ +statement ok +create table t1 (a int); + +statement ok +create source s1 (a int) with (connector='datagen'); + +statement ok +create materialized view mv1 as select t1.a from t1 join s1 on t1.a = s1.a; + +statement ok +create materialized view mv2 as select * from mv1; + +statement ok +create sink sink1 from mv2 with (connector='blackhole'); + + +# equivalent to: +# select objid::regclass, refobjid::regclass from rw_depend; +query TT rowsort +select r1.name, r2.name relname +from rw_depend d +join rw_relations r1 on d.objid = r1.id +join rw_relations r2 on d.refobjid = r2.id; +---- +mv1 s1 +mv1 t1 +mv2 mv1 +sink1 mv2 + +statement ok +drop sink sink1; + +statement ok +drop materialized view mv2; + +statement ok +drop materialized view mv1; + +statement ok +drop source s1; + +statement ok +drop table t1; diff --git a/e2e_test/batch/catalog/sysinfo.slt.part b/e2e_test/batch/catalog/sysinfo.slt.part index dd6d888b8beb0..5c9aea2fb6103 100644 --- a/e2e_test/batch/catalog/sysinfo.slt.part +++ b/e2e_test/batch/catalog/sysinfo.slt.part @@ -50,7 +50,7 @@ CREATE INDEX tab_idx ON tab(a, (b + c + (1 + 1))) INCLUDE(d) CREATE INDEX tab_id query T select pg_get_indexdef('tab_idx'::regclass, 1, true), pg_get_indexdef('tab_idx'::regclass, 2, true); ---- -a ((b + c) + (1:Int32 + 1:Int32)) +a ((b + c) + 2:Int32) query T select pg_get_indexdef('tab_idx'::regclass, 3, true); @@ -84,3 +84,7 @@ drop index tab_idx; statement ok drop table tab; + +query T +select * from pg_catalog.pg_sequences; +---- diff --git a/e2e_test/batch/catalog/version.slt.part b/e2e_test/batch/catalog/version.slt.part index b2ba9e2a877c5..dc3e0399b1e6a 100644 --- a/e2e_test/batch/catalog/version.slt.part +++ b/e2e_test/batch/catalog/version.slt.part @@ -1,4 +1,4 @@ query T -select substring(version() from 1 for 14); +select substring(version() from 1 for 16); ---- -PostgreSQL 9.5 +PostgreSQL 13.14 diff --git a/e2e_test/batch/functions/setting.slt.part b/e2e_test/batch/functions/setting.slt.part index 77d1d80e46590..233399d80a025 100644 --- a/e2e_test/batch/functions/setting.slt.part +++ b/e2e_test/batch/functions/setting.slt.part @@ -1,12 +1,12 @@ query T SELECT current_setting('server_version'); ---- -9.5.0 +13.14.0 query I -SELECT CAST(current_setting('server_version_num') AS INT) / 100 AS version; +SELECT current_setting('server_version_num') AS version; ---- -905 +130014 query T SELECT set_config('client_min_messages', 'warning', false); diff --git a/e2e_test/batch/transaction/now.slt b/e2e_test/batch/transaction/now.slt index 4f8d317f04261..bb717dfc6ed1e 100644 --- a/e2e_test/batch/transaction/now.slt +++ b/e2e_test/batch/transaction/now.slt @@ -33,11 +33,12 @@ except select * from mv; ---- -query T -select * from mv -except -select * from v; ----- +# temporarily disable the check before is resolved https://github.com/risingwavelabs/risingwave/issues/15117 +## query T +## select * from mv +## except +## select * from v; +## ---- statement ok commit; diff --git a/e2e_test/batch/types/interval.slt.part b/e2e_test/batch/types/interval.slt.part index b6eef0915ab3c..629f2c77ef8b7 100644 --- a/e2e_test/batch/types/interval.slt.part +++ b/e2e_test/batch/types/interval.slt.part @@ -185,6 +185,24 @@ select v / d from t; 00:01:19.101562 00:03:57.304688 +query T +select 'P1Y2M3DT4H5M6S'::interval; +---- +1 year 2 mons 3 days 04:05:06 + +query T +select 'P0Y0M0DT0H0M0S'::interval; +---- +00:00:00 + +query T +select 'P12Y2M30DT4H5M0.1234567S'::interval; +---- +12 years 2 mons 30 days 04:05:00.123456 + +statement error +select 'P0Y0M0DT0H0M0S1'::interval; + # The following is an overflow bug present in PostgreSQL 15.2 # Their `days` overflows to a negative value, leading to the latter smaller # than the former. We report an error in this case. diff --git a/e2e_test/ddl/alter_parallelism.slt b/e2e_test/ddl/alter_parallelism.slt index e030b44273575..025496ca1c571 100644 --- a/e2e_test/ddl/alter_parallelism.slt +++ b/e2e_test/ddl/alter_parallelism.slt @@ -22,7 +22,7 @@ create table t (v int); query T select parallelism from table_parallelism where name = 't'; ---- -AUTO +ADAPTIVE statement ok alter table t set parallelism = 2; @@ -39,12 +39,12 @@ select parallelism from fragment_parallelism where table_name = 't'; 2 statement ok -alter table t set parallelism = auto; +alter table t set parallelism = adaptive; query T select parallelism from table_parallelism where name = 't'; ---- -AUTO +ADAPTIVE statement ok create materialized view m_simple as select * from t; @@ -52,7 +52,7 @@ create materialized view m_simple as select * from t; query T select parallelism from mview_parallelism where name = 'm_simple'; ---- -AUTO +ADAPTIVE statement ok alter materialized view m_simple set parallelism = 3; @@ -68,7 +68,7 @@ create materialized view m_join as select t1.v as t1v, t2.v as t2v from t t1, t query T select parallelism from mview_parallelism where name = 'm_join'; ---- -AUTO +ADAPTIVE statement ok alter materialized view m_join set parallelism = 3; @@ -84,7 +84,7 @@ create sink s as select t1.v as t1v, t2.v as t2v from t t1, t t2 where t1.v = t2 query T select parallelism from sink_parallelism where name = 's'; ---- -AUTO +ADAPTIVE statement ok alter sink s set parallelism = 4; @@ -121,7 +121,7 @@ statement ok drop table t; statement ok -set streaming_parallelism to auto; +set streaming_parallelism to adaptive; statement ok create table t (v1 int); @@ -129,7 +129,7 @@ create table t (v1 int); query T select parallelism from table_parallelism where name = 't'; ---- -AUTO +ADAPTIVE statement ok drop table t; @@ -143,7 +143,7 @@ create table t (v1 int); query T select parallelism from table_parallelism where name = 't'; ---- -AUTO +ADAPTIVE statement ok drop table t; diff --git a/e2e_test/ddl/alter_rename.slt b/e2e_test/ddl/alter_rename.slt index 11b7007f72bf0..5171f7f3cdad1 100644 --- a/e2e_test/ddl/alter_rename.slt +++ b/e2e_test/ddl/alter_rename.slt @@ -229,7 +229,7 @@ DROP SCHEMA schema1; statement ok DROP SINK sink1; -statement error other relation\(s\) depend on it +statement error Permission denied DROP VIEW v5; statement ok diff --git a/e2e_test/ddl/dependency_check.slt b/e2e_test/ddl/dependency_check.slt index 88a89975d07ef..413579bb13e7b 100644 --- a/e2e_test/ddl/dependency_check.slt +++ b/e2e_test/ddl/dependency_check.slt @@ -16,7 +16,7 @@ create index i_b1 on b(b1); statement ok create materialized view mv1 as select * from a join b on a.a1 = b.b1; -statement error other relation\(s\) depend on it +statement error Permission denied drop index i_a1; statement ok @@ -25,7 +25,7 @@ drop materialized view mv1; statement ok create materialized view mv2 as with ctx as (select a1 from a) select b1 from b; -statement error other relation\(s\) depend on it +statement error Permission denied drop table a; statement ok @@ -48,13 +48,13 @@ create view v2 as select * from v; statement ok create materialized view mv3 as select * from v2; -statement error other relation\(s\) depend on it +statement error Permission denied drop source src; -statement error other relation\(s\) depend on it +statement error Permission denied drop view v; -statement error other relation\(s\) depend on it +statement error Permission denied drop view v2; statement ok diff --git a/e2e_test/ddl/search_path.slt b/e2e_test/ddl/search_path.slt index 06db7f3f45c90..e01ed5d602936 100644 --- a/e2e_test/ddl/search_path.slt +++ b/e2e_test/ddl/search_path.slt @@ -76,6 +76,11 @@ select a from test order by a; 1 2 +# Issue #15195 +# index shall be created in `search_path_test2` (same as table) rather than `search_path_test1` (first in path) +statement ok +create index if not exists index1_test_a on test(a); + statement ok drop table test; diff --git a/e2e_test/ddl/table/table.slt.part b/e2e_test/ddl/table/table.slt.part index 2e7c744ba2536..d814a5beee10c 100644 --- a/e2e_test/ddl/table/table.slt.part +++ b/e2e_test/ddl/table/table.slt.part @@ -258,4 +258,4 @@ statement ok drop table t1; statement ok -drop table t2; +drop table t2; \ No newline at end of file diff --git a/e2e_test/error_ui/simple/main.slt b/e2e_test/error_ui/simple/main.slt index b4cebbdfeff70..50eedcf4529cf 100644 --- a/e2e_test/error_ui/simple/main.slt +++ b/e2e_test/error_ui/simple/main.slt @@ -9,15 +9,27 @@ Near "selet" statement error -create function int_42() returns int as int_42 using link 'localhost:8815'; +create function int_42() returns int as int_42 using link '555.0.0.1:8815'; +---- +db error: ERROR: Failed to run the query + +Caused by: + Flight service error: invalid address: 555.0.0.1:8815, err: failed to parse address: http://555.0.0.1:8815: invalid IPv4 address + + +statement error +create function int_42() returns int as int_42 using link '55.55.55.55:5555'; ---- db error: ERROR: Failed to run the query Caused by these errors (recent errors listed first): - 1: failed to connect to UDF service - 2: transport error - 3: error trying to connect - 4: invalid URL, scheme is missing + 1: failed to check UDF signature + 2: failed to send requests to UDF service + 3: status: Unavailable, message: "error trying to connect: tcp connect error: deadline has elapsed", details: [], metadata: MetadataMap { headers: {} } + 4: transport error + 5: error trying to connect + 6: tcp connect error + 7: deadline has elapsed statement error @@ -27,7 +39,7 @@ db error: ERROR: Failed to run the query Caused by these errors (recent errors listed first): 1: gRPC request to meta service failed: Internal error - 2: SystemParams error: unrecognized system param "not_exist_key" + 2: SystemParams error: unrecognized system parameter "not_exist_key" query error diff --git a/e2e_test/iceberg/start_spark_connect_server.sh b/e2e_test/iceberg/start_spark_connect_server.sh index 827b8dde86179..9e263e70c8e36 100755 --- a/e2e_test/iceberg/start_spark_connect_server.sh +++ b/e2e_test/iceberg/start_spark_connect_server.sh @@ -8,17 +8,20 @@ PACKAGES="$PACKAGES,org.apache.spark:spark-connect_2.12:$SPARK_VERSION" SPARK_FILE="spark-${SPARK_VERSION}-bin-hadoop3.tgz" - -wget https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/$SPARK_FILE -tar -xzf $SPARK_FILE --no-same-owner +if [ ! -d "spark-${SPARK_VERSION}-bin-hadoop3" ];then + wget https://dlcdn.apache.org/spark/spark-${SPARK_VERSION}/$SPARK_FILE + tar -xzf $SPARK_FILE --no-same-owner +fi ./spark-${SPARK_VERSION}-bin-hadoop3/sbin/start-connect-server.sh --packages $PACKAGES \ --master local[3] \ --conf spark.driver.bindAddress=0.0.0.0 \ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \ --conf spark.sql.catalog.demo.type=hadoop \ --conf spark.sql.catalog.demo.warehouse=s3a://icebergdata/demo \ --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://127.0.0.1:9301 \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.path.style.access=true \ --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=hummockadmin \ --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=hummockadmin \ --conf spark.sql.defaultCatalog=demo @@ -29,4 +32,4 @@ while ! nc -z localhost 15002; do sleep 1 # wait for 1/10 of the second before check again done -echo "Spark connect server launched" \ No newline at end of file +echo "Spark connect server launched" diff --git a/e2e_test/iceberg/test_case/cdc/load.slt b/e2e_test/iceberg/test_case/cdc/load.slt index 2ac8ab2d61f25..e9f1d815d20cb 100644 --- a/e2e_test/iceberg/test_case/cdc/load.slt +++ b/e2e_test/iceberg/test_case/cdc/load.slt @@ -7,8 +7,8 @@ create source mysql_mydb with ( port = '3306', username = 'root', password = '123456', - database.name = 'my@db', - server.id = '2' + database.name = 'mydb', + server.id = '5085' ); statement ok @@ -16,7 +16,7 @@ create table products ( id INT, name STRING, description STRING, PRIMARY KEY (id) -) FROM mysql_mydb TABLE 'my@db.products'; +) FROM mysql_mydb TABLE 'mydb.products'; statement ok @@ -24,8 +24,9 @@ CREATE SINK s1 AS select * from products WITH ( connector = 'iceberg', type = 'upsert', force_append_only = 'false', - database.name = 'demo', - table.name = 'demo_db.demo_table', + catalog.name = 'demo', + database.name = 'demo_db', + table.name = 'demo_table', catalog.type = 'storage', warehouse.path = 's3://icebergdata/demo', s3.endpoint = 'http://127.0.0.1:9301', @@ -35,13 +36,9 @@ CREATE SINK s1 AS select * from products WITH ( primary_key = 'id' ); -statement ok -flush; +sleep 20s query I select count(*) from products; ---- 8 - -statement ok -flush; diff --git a/e2e_test/iceberg/test_case/cdc/mysql_cdc.sql b/e2e_test/iceberg/test_case/cdc/mysql_cdc.sql index b7b6f13af83cf..f95c6c2c8d4a8 100644 --- a/e2e_test/iceberg/test_case/cdc/mysql_cdc.sql +++ b/e2e_test/iceberg/test_case/cdc/mysql_cdc.sql @@ -1,7 +1,7 @@ -DROP DATABASE IF EXISTS `my@db`; -CREATE DATABASE `my@db`; +DROP DATABASE IF EXISTS `mydb`; +CREATE DATABASE `mydb`; -USE `my@db`; +USE `mydb`; CREATE TABLE products ( id INTEGER NOT NULL AUTO_INCREMENT PRIMARY KEY, diff --git a/e2e_test/iceberg/test_case/cdc/mysql_cdc_insert.sql b/e2e_test/iceberg/test_case/cdc/mysql_cdc_insert.sql index 641d6220ea8dc..c7dc50316d3cb 100644 --- a/e2e_test/iceberg/test_case/cdc/mysql_cdc_insert.sql +++ b/e2e_test/iceberg/test_case/cdc/mysql_cdc_insert.sql @@ -1,4 +1,4 @@ -USE `my@db`; +USE `mydb`; INSERT INTO products VALUES (default,"109","109"), (default,"110","110"), diff --git a/e2e_test/iceberg/test_case/iceberg_sink_append_only.slt b/e2e_test/iceberg/test_case/iceberg_sink_append_only.slt index 5f847eaa30a7e..69cd931dbf836 100644 --- a/e2e_test/iceberg/test_case/iceberg_sink_append_only.slt +++ b/e2e_test/iceberg/test_case/iceberg_sink_append_only.slt @@ -23,8 +23,9 @@ CREATE SINK s6 AS select * from mv6 WITH ( connector = 'iceberg', type = 'append-only', force_append_only = 'true', - database.name = 'demo', - table.name = 'demo_db.demo_table', + database.name = 'demo_db', + table.name = 'demo_table', + catalog.name = 'demo', catalog.type = 'storage', warehouse.path = 's3://icebergdata/demo', s3.endpoint = 'http://127.0.0.1:9301', @@ -43,6 +44,8 @@ INSERT INTO t6 VALUES statement ok FLUSH; +sleep 5s + statement ok INSERT INTO t6 VALUES (5, 5, 5000, 5.5, 5.55, '5-5', true, '2022-03-15', '2022-03-15 05:00:00Z'::timestamptz, '2022-03-15 05:00:00'); @@ -50,6 +53,8 @@ INSERT INTO t6 VALUES statement ok FLUSH; +sleep 5s + statement ok DROP SINK s6; diff --git a/e2e_test/iceberg/test_case/iceberg_sink_upsert.slt b/e2e_test/iceberg/test_case/iceberg_sink_upsert.slt index 646a39cc08e28..1d0be3d56463e 100644 --- a/e2e_test/iceberg/test_case/iceberg_sink_upsert.slt +++ b/e2e_test/iceberg/test_case/iceberg_sink_upsert.slt @@ -2,18 +2,19 @@ statement ok set streaming_parallelism=4; statement ok -CREATE TABLE t6 (id int, v1 int primary key, v2 bigint, v3 varchar); +CREATE TABLE t6 (id int, v1 int primary key, v2 bigint, v3 varchar, v4 date); statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK s6 AS select mv6.id as id, mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH ( +CREATE SINK s6 AS select mv6.id as id, mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3, mv6.v4 as v4 from mv6 WITH ( connector = 'iceberg', type = 'upsert', force_append_only = 'false', - database.name = 'demo', - table.name = 'demo_db.demo_table', + catalog.name = 'demo', + database.name = 'demo_db', + table.name = 'demo_table', catalog.type = 'storage', warehouse.path = 's3://icebergdata/demo', s3.endpoint = 'http://127.0.0.1:9301', @@ -24,17 +25,31 @@ CREATE SINK s6 AS select mv6.id as id, mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 ); statement ok -INSERT INTO t6 VALUES (1, 1, 2, '1-2'), (1, 2, 2, '2-2'), (1, 3, 2, '3-2'), (1, 5, 2, '5-2'), (1, 8, 2, '8-2'), (1, 13, 2, '13-2'), (1, 21, 2, '21-2'); +INSERT INTO t6 VALUES (1, 1, 2, '1-2', '2022-03-11'), (1, 2, 2, '2-2', '2022-03-12'), (1, 3, 2, '3-2', '2022-03-13'), (1, 5, 2, '5-2', '2022-03-15'), (1, 8, 2, '8-2', '2022-03-18'), (1, 13, 2, '13-2', '2022-03-13'), (1, 21, 2, '21-2', '2022-03-21'); statement ok FLUSH; +sleep 5s + statement ok -INSERT INTO t6 VALUES (1, 1, 50, '1-50'); +INSERT INTO t6 VALUES (1, 1, 50, '1-50', '2022-03-11'); statement ok FLUSH; +sleep 10s + +query I +select count(*) from t6; +---- +7 + +statement ok +FLUSH; + +sleep 10s + statement ok DROP SINK s6; diff --git a/e2e_test/iceberg/test_case/no_partition_append_only.toml b/e2e_test/iceberg/test_case/no_partition_append_only.toml index 211407644abec..4b2b80221d9e0 100644 --- a/e2e_test/iceberg/test_case/no_partition_append_only.toml +++ b/e2e_test/iceberg/test_case/no_partition_append_only.toml @@ -13,7 +13,7 @@ init_sqls = [ v_date date, v_timestamp timestamp, v_ts_ntz timestamp_ntz - ) TBLPROPERTIES ('format-version'='2'); + ) USING iceberg TBLPROPERTIES ('format-version'='2'); ''' ] diff --git a/e2e_test/iceberg/test_case/no_partition_upsert.toml b/e2e_test/iceberg/test_case/no_partition_upsert.toml index 0e0215d37465d..cd380c53da88c 100644 --- a/e2e_test/iceberg/test_case/no_partition_upsert.toml +++ b/e2e_test/iceberg/test_case/no_partition_upsert.toml @@ -6,7 +6,8 @@ init_sqls = [ id int, v1 int, v2 long, - v3 string + v3 string, + v4 date ) USING iceberg TBLPROPERTIES ('format-version'='2'); ''' @@ -19,13 +20,13 @@ verify_schema = ['int','int','long','string'] verify_sql = 'SELECT * FROM demo_db.demo_table ORDER BY id, v1 ASC' verify_data = """ -1,1,50,1-50 -1,2,2,2-2 -1,3,2,3-2 -1,5,2,5-2 -1,8,2,8-2 -1,13,2,13-2 -1,21,2,21-2 +1,1,50,1-50,2022-03-11 +1,2,2,2-2,2022-03-12 +1,3,2,3-2,2022-03-13 +1,5,2,5-2,2022-03-15 +1,8,2,8-2,2022-03-18 +1,13,2,13-2,2022-03-13 +1,21,2,21-2,2022-03-21 """ drop_sqls = [ diff --git a/e2e_test/iceberg/test_case/partition_append_only.toml b/e2e_test/iceberg/test_case/partition_append_only.toml index 4721ef11c5ba6..e1949319211cf 100644 --- a/e2e_test/iceberg/test_case/partition_append_only.toml +++ b/e2e_test/iceberg/test_case/partition_append_only.toml @@ -14,7 +14,7 @@ init_sqls = [ v_timestamp timestamp, v_ts_ntz timestamp_ntz ) - PARTITIONED BY (v_int,v_long,v_float,v_double,v_varchar,v_bool,v_date,v_timestamp,v_ts_ntz) + PARTITIONED BY (v_int,bucket(10,v_long),truncate(30,v_long),years(v_date),months(v_timestamp),days(v_ts_ntz)) TBLPROPERTIES ('format-version'='2'); ''' ] diff --git a/e2e_test/iceberg/test_case/partition_upsert.toml b/e2e_test/iceberg/test_case/partition_upsert.toml index d95178ed893fa..ea027f18ecdbe 100644 --- a/e2e_test/iceberg/test_case/partition_upsert.toml +++ b/e2e_test/iceberg/test_case/partition_upsert.toml @@ -6,9 +6,10 @@ init_sqls = [ id int, v1 int, v2 long, - v3 string + v3 string, + v4 date ) USING iceberg - PARTITIONED BY (v1,v2) + PARTITIONED BY (v1,v2,truncate(2,v3)) TBLPROPERTIES ('format-version'='2'); ''' ] @@ -20,13 +21,13 @@ verify_schema = ['int','int','long','string'] verify_sql = 'SELECT * FROM demo_db.demo_table ORDER BY id, v1 ASC' verify_data = """ -1,1,50,1-50 -1,2,2,2-2 -1,3,2,3-2 -1,5,2,5-2 -1,8,2,8-2 -1,13,2,13-2 -1,21,2,21-2 +1,1,50,1-50,2022-03-11 +1,2,2,2-2,2022-03-12 +1,3,2,3-2,2022-03-13 +1,5,2,5-2,2022-03-15 +1,8,2,8-2,2022-03-18 +1,13,2,13-2,2022-03-13 +1,21,2,21-2,2022-03-21 """ drop_sqls = [ diff --git a/e2e_test/iceberg/test_case/range_partition_append_only.toml b/e2e_test/iceberg/test_case/range_partition_append_only.toml new file mode 100644 index 0000000000000..d0e270210dffe --- /dev/null +++ b/e2e_test/iceberg/test_case/range_partition_append_only.toml @@ -0,0 +1,40 @@ +init_sqls = [ + 'CREATE SCHEMA IF NOT EXISTS demo_db', + 'DROP TABLE IF EXISTS demo_db.demo_table', + ''' + CREATE TABLE demo_db.demo_table ( + id long, + v_int int, + v_long long, + v_float float, + v_double double, + v_varchar string, + v_bool boolean, + v_date date, + v_timestamp timestamp, + v_ts_ntz timestamp_ntz + ) + PARTITIONED BY (years(v_date),months(v_timestamp),days(v_ts_ntz)) + TBLPROPERTIES ('format-version'='2'); + ''' +] + +slt = 'test_case/iceberg_sink_append_only.slt' + +verify_schema = ['long', 'int', 'long', 'float', 'double', 'string', 'boolean', 'date', 'timestamp', 'timestamp_ntz'] + +verify_sql = 'SELECT * FROM demo_db.demo_table ORDER BY id ASC' + + +verify_data = """ +1,1,1000,1.1,1.11,1-1,true,2022-03-11,2022-03-11 01:00:00+00:00,2022-03-11 01:00:00 +2,2,2000,2.2,2.22,2-2,false,2022-03-12,2022-03-12 02:00:00+00:00,2022-03-12 02:00:00 +3,3,3000,3.3,3.33,3-3,true,2022-03-13,2022-03-13 03:00:00+00:00,2022-03-13 03:00:00 +4,4,4000,4.4,4.44,4-4,false,2022-03-14,2022-03-14 04:00:00+00:00,2022-03-14 04:00:00 +5,5,5000,5.5,5.55,5-5,true,2022-03-15,2022-03-15 05:00:00+00:00,2022-03-15 05:00:00 +""" + +drop_sqls = [ + 'DROP TABLE IF EXISTS demo_db.demo_table', + 'DROP SCHEMA IF EXISTS demo_db' +] diff --git a/e2e_test/iceberg/test_case/range_partition_upsert.toml b/e2e_test/iceberg/test_case/range_partition_upsert.toml new file mode 100644 index 0000000000000..ac081d6edeaba --- /dev/null +++ b/e2e_test/iceberg/test_case/range_partition_upsert.toml @@ -0,0 +1,36 @@ +init_sqls = [ + 'CREATE SCHEMA IF NOT EXISTS demo_db', + 'DROP TABLE IF EXISTS demo_db.demo_table', + ''' + CREATE TABLE demo_db.demo_table ( + id int, + v1 int, + v2 long, + v3 string, + v4 date + ) USING iceberg + PARTITIONED BY (days(v4)) + TBLPROPERTIES ('format-version'='2'); + ''' +] + +slt = 'test_case/iceberg_sink_upsert.slt' + +verify_schema = ['int','int','long','string'] + +verify_sql = 'SELECT * FROM demo_db.demo_table ORDER BY id, v1 ASC' + +verify_data = """ +1,1,50,1-50,2022-03-11 +1,2,2,2-2,2022-03-12 +1,3,2,3-2,2022-03-13 +1,5,2,5-2,2022-03-15 +1,8,2,8-2,2022-03-18 +1,13,2,13-2,2022-03-13 +1,21,2,21-2,2022-03-21 +""" + +drop_sqls = [ + 'DROP TABLE IF EXISTS demo_db.demo_table', + 'DROP SCHEMA IF EXISTS demo_db' +] diff --git a/e2e_test/over_window/generated/batch/basic/cross_check.slt.part b/e2e_test/over_window/generated/batch/basic/cross_check.slt.part index 34ef01e43b837..764a60cfba86e 100644 --- a/e2e_test/over_window/generated/batch/basic/cross_check.slt.part +++ b/e2e_test/over_window/generated/batch/basic/cross_check.slt.part @@ -21,6 +21,16 @@ select from v_a natural join v_c; ---- +query i +select + id, out1, out10, out11 +from v_a_d +except +select + id, out1, out10, out11 +from v_a natural join v_d; +---- + query i select id, out3, out4, out7, out8, out9 @@ -31,6 +41,26 @@ select from v_b natural join v_c; ---- +query i +select + id, out3, out4, out10, out11 +from v_b_d +except +select + id, out3, out4, out10, out11 +from v_b natural join v_d; +---- + +query i +select + id, out7, out8, out9, out10, out11 +from v_c_d +except +select + id, out7, out8, out9, out10, out11 +from v_c natural join v_d; +---- + query i select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 @@ -40,3 +70,13 @@ select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 from v_a natural join v_b natural join v_c; ---- + +query i +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a_b_c_d +except +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a natural join v_b natural join v_c natural join v_d; +---- diff --git a/e2e_test/over_window/generated/batch/basic/mod.slt.part b/e2e_test/over_window/generated/batch/basic/mod.slt.part index a8c74b16bf790..1c5ff0f9b460c 100644 --- a/e2e_test/over_window/generated/batch/basic/mod.slt.part +++ b/e2e_test/over_window/generated/batch/basic/mod.slt.part @@ -35,6 +35,14 @@ select * from v_c order by id; 100003 100 208 2 723 807 723 NULL NULL NULL NULL 100004 103 200 2 702 808 702 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2124 +100003 100 208 2 723 807 1 2124 +100004 103 200 2 702 808 2 702 + include ./cross_check.slt.part statement ok @@ -72,6 +80,16 @@ select * from v_c order by id; 100005 100 200 3 717 810 717 700 700 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2841 +100003 100 208 2 723 807 1 2841 +100004 103 200 2 702 808 2 702 +100005 100 200 3 717 810 1 2140 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -113,6 +131,16 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 723 701 806 806 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2940 +100002 100 200 2 799 806 1 2940 +100003 100 200 2 723 807 1 2940 +100004 103 200 2 702 808 2 702 +100005 100 200 1 717 810 1 2940 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -139,6 +167,13 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 701 701 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 1418 +100005 100 200 1 717 810 1 1418 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part include ./teardown.slt.part diff --git a/e2e_test/over_window/generated/batch/basic/setup.slt.part b/e2e_test/over_window/generated/batch/basic/setup.slt.part index 2ffc1b055334d..578a43df88113 100644 --- a/e2e_test/over_window/generated/batch/basic/setup.slt.part +++ b/e2e_test/over_window/generated/batch/basic/setup.slt.part @@ -40,6 +40,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +# range frame +statement ok +create view v_d as +select + * + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create view v_a_b as select @@ -59,6 +68,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create view v_a_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create view v_b_c as select @@ -70,6 +88,27 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create view v_b_d as +select + * + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + +statement ok +create view v_c_d as +select + * + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create view v_a_b_c as select @@ -84,3 +123,20 @@ select , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; + +statement ok +create view v_a_b_c_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , avg(v1) over (partition by p1) as out2 + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , lag(v1, 0) over (partition by p1 order by id) as out5 + , lag(v1, 1) over (partition by p1, p2 order by id) as out6 + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; diff --git a/e2e_test/over_window/generated/batch/basic/teardown.slt.part b/e2e_test/over_window/generated/batch/basic/teardown.slt.part index 97f416dd49c9e..b0314f2ce346b 100644 --- a/e2e_test/over_window/generated/batch/basic/teardown.slt.part +++ b/e2e_test/over_window/generated/batch/basic/teardown.slt.part @@ -9,17 +9,32 @@ drop view v_b; statement ok drop view v_c; +statement ok +drop view v_d; + statement ok drop view v_a_b; +statement ok +drop view v_a_c; + +statement ok +drop view v_a_d; + statement ok drop view v_b_c; statement ok -drop view v_a_c; +drop view v_b_d; + +statement ok +drop view v_c_d; statement ok drop view v_a_b_c; +statement ok +drop view v_a_b_c_d; + statement ok drop table t; diff --git a/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part b/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part index 34ef01e43b837..764a60cfba86e 100644 --- a/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/cross_check.slt.part @@ -21,6 +21,16 @@ select from v_a natural join v_c; ---- +query i +select + id, out1, out10, out11 +from v_a_d +except +select + id, out1, out10, out11 +from v_a natural join v_d; +---- + query i select id, out3, out4, out7, out8, out9 @@ -31,6 +41,26 @@ select from v_b natural join v_c; ---- +query i +select + id, out3, out4, out10, out11 +from v_b_d +except +select + id, out3, out4, out10, out11 +from v_b natural join v_d; +---- + +query i +select + id, out7, out8, out9, out10, out11 +from v_c_d +except +select + id, out7, out8, out9, out10, out11 +from v_c natural join v_d; +---- + query i select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 @@ -40,3 +70,13 @@ select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 from v_a natural join v_b natural join v_c; ---- + +query i +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a_b_c_d +except +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a natural join v_b natural join v_c natural join v_d; +---- diff --git a/e2e_test/over_window/generated/streaming/basic/mod.slt.part b/e2e_test/over_window/generated/streaming/basic/mod.slt.part index a8c74b16bf790..1c5ff0f9b460c 100644 --- a/e2e_test/over_window/generated/streaming/basic/mod.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/mod.slt.part @@ -35,6 +35,14 @@ select * from v_c order by id; 100003 100 208 2 723 807 723 NULL NULL NULL NULL 100004 103 200 2 702 808 702 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2124 +100003 100 208 2 723 807 1 2124 +100004 103 200 2 702 808 2 702 + include ./cross_check.slt.part statement ok @@ -72,6 +80,16 @@ select * from v_c order by id; 100005 100 200 3 717 810 717 700 700 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2841 +100003 100 208 2 723 807 1 2841 +100004 103 200 2 702 808 2 702 +100005 100 200 3 717 810 1 2140 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -113,6 +131,16 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 723 701 806 806 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2940 +100002 100 200 2 799 806 1 2940 +100003 100 200 2 723 807 1 2940 +100004 103 200 2 702 808 2 702 +100005 100 200 1 717 810 1 2940 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -139,6 +167,13 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 701 701 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 1418 +100005 100 200 1 717 810 1 1418 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part include ./teardown.slt.part diff --git a/e2e_test/over_window/generated/streaming/basic/setup.slt.part b/e2e_test/over_window/generated/streaming/basic/setup.slt.part index cc46c4066f0f4..b4457cde0419b 100644 --- a/e2e_test/over_window/generated/streaming/basic/setup.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/setup.slt.part @@ -40,6 +40,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +# range frame +statement ok +create materialized view v_d as +select + * + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create materialized view v_a_b as select @@ -59,6 +68,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create materialized view v_a_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create materialized view v_b_c as select @@ -70,6 +88,27 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create materialized view v_b_d as +select + * + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + +statement ok +create materialized view v_c_d as +select + * + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create materialized view v_a_b_c as select @@ -84,3 +123,20 @@ select , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; + +statement ok +create materialized view v_a_b_c_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , avg(v1) over (partition by p1) as out2 + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , lag(v1, 0) over (partition by p1 order by id) as out5 + , lag(v1, 1) over (partition by p1, p2 order by id) as out6 + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; diff --git a/e2e_test/over_window/generated/streaming/basic/teardown.slt.part b/e2e_test/over_window/generated/streaming/basic/teardown.slt.part index 2089fefcac249..041677a88c268 100644 --- a/e2e_test/over_window/generated/streaming/basic/teardown.slt.part +++ b/e2e_test/over_window/generated/streaming/basic/teardown.slt.part @@ -9,17 +9,32 @@ drop materialized view v_b; statement ok drop materialized view v_c; +statement ok +drop materialized view v_d; + statement ok drop materialized view v_a_b; +statement ok +drop materialized view v_a_c; + +statement ok +drop materialized view v_a_d; + statement ok drop materialized view v_b_c; statement ok -drop materialized view v_a_c; +drop materialized view v_b_d; + +statement ok +drop materialized view v_c_d; statement ok drop materialized view v_a_b_c; +statement ok +drop materialized view v_a_b_c_d; + statement ok drop table t; diff --git a/e2e_test/over_window/templates/basic/cross_check.slt.part b/e2e_test/over_window/templates/basic/cross_check.slt.part index b2a8041257143..0731d52637f45 100644 --- a/e2e_test/over_window/templates/basic/cross_check.slt.part +++ b/e2e_test/over_window/templates/basic/cross_check.slt.part @@ -19,6 +19,16 @@ select from v_a natural join v_c; ---- +query i +select + id, out1, out10, out11 +from v_a_d +except +select + id, out1, out10, out11 +from v_a natural join v_d; +---- + query i select id, out3, out4, out7, out8, out9 @@ -29,6 +39,26 @@ select from v_b natural join v_c; ---- +query i +select + id, out3, out4, out10, out11 +from v_b_d +except +select + id, out3, out4, out10, out11 +from v_b natural join v_d; +---- + +query i +select + id, out7, out8, out9, out10, out11 +from v_c_d +except +select + id, out7, out8, out9, out10, out11 +from v_c natural join v_d; +---- + query i select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 @@ -38,3 +68,13 @@ select id, out1, out2, out3, out4, out5, out6, out7, out8, out9 from v_a natural join v_b natural join v_c; ---- + +query i +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a_b_c_d +except +select + id, out1, out2, out3, out4, out5, out6, out7, out8, out9, out10, out11 +from v_a natural join v_b natural join v_c natural join v_d; +---- diff --git a/e2e_test/over_window/templates/basic/mod.slt.part b/e2e_test/over_window/templates/basic/mod.slt.part index 421f5a911f468..d2ed6fdc1e107 100644 --- a/e2e_test/over_window/templates/basic/mod.slt.part +++ b/e2e_test/over_window/templates/basic/mod.slt.part @@ -33,6 +33,14 @@ select * from v_c order by id; 100003 100 208 2 723 807 723 NULL NULL NULL NULL 100004 103 200 2 702 808 702 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2124 +100003 100 208 2 723 807 1 2124 +100004 103 200 2 702 808 2 702 + include ./cross_check.slt.part statement ok @@ -70,6 +78,16 @@ select * from v_c order by id; 100005 100 200 3 717 810 717 700 700 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2124 +100002 100 200 2 700 806 1 2841 +100003 100 208 2 723 807 1 2841 +100004 103 200 2 702 808 2 702 +100005 100 200 3 717 810 1 2140 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -111,6 +129,16 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 723 701 806 806 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 2940 +100002 100 200 2 799 806 1 2940 +100003 100 200 2 723 807 1 2940 +100004 103 200 2 702 808 2 702 +100005 100 200 1 717 810 1 2940 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part statement ok @@ -137,6 +165,13 @@ select * from v_c order by id; 100005 100 200 1 717 810 717 701 701 NULL NULL 100006 105 204 5 703 828 703 NULL NULL NULL NULL +query iiiiiiii +select * from v_d order by id; +---- +100001 100 200 1 701 805 1 1418 +100005 100 200 1 717 810 1 1418 +100006 105 204 5 703 828 5 703 + include ./cross_check.slt.part include ./teardown.slt.part diff --git a/e2e_test/over_window/templates/basic/setup.slt.part b/e2e_test/over_window/templates/basic/setup.slt.part index d989d50029430..f08fc7e4ea20d 100644 --- a/e2e_test/over_window/templates/basic/setup.slt.part +++ b/e2e_test/over_window/templates/basic/setup.slt.part @@ -38,6 +38,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +# range frame +statement ok +create $view_type v_d as +select + * + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create $view_type v_a_b as select @@ -57,6 +66,15 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create $view_type v_a_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create $view_type v_b_c as select @@ -68,6 +86,27 @@ select , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; +statement ok +create $view_type v_b_d as +select + * + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + +statement ok +create $view_type v_c_d as +select + * + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; + statement ok create $view_type v_a_b_c as select @@ -82,3 +121,20 @@ select , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 from t; + +statement ok +create $view_type v_a_b_c_d as +select + * + , first_value(v1) over (partition by p1, p2 order by time, id rows 3 preceding) as out1 + , avg(v1) over (partition by p1) as out2 + , sum(v1) over (partition by p1, p2 order by time, id rows between unbounded preceding and current row) as out3 + , min(v1) over (partition by p1, p2 order by time, id rows between current row and unbounded following) as out4 + , lag(v1, 0) over (partition by p1 order by id) as out5 + , lag(v1, 1) over (partition by p1, p2 order by id) as out6 + , lag(v1) over (partition by p1, p2 order by time, id) as out7 + , lead(v2, 1) over (partition by p1, p2 order by time, id) as out8 + , lead(v2, 2) over (partition by p1, p2 order by v1, v2) as out9 + , last_value(time) over (partition by p1 order by time desc range between current row and 2 following) as out10 + , sum(v1) over (partition by p1 order by time range between 1 preceding and 1 following) as out11 +from t; diff --git a/e2e_test/over_window/templates/basic/teardown.slt.part b/e2e_test/over_window/templates/basic/teardown.slt.part index 89915395703ad..a9bf9252c385a 100644 --- a/e2e_test/over_window/templates/basic/teardown.slt.part +++ b/e2e_test/over_window/templates/basic/teardown.slt.part @@ -7,17 +7,32 @@ drop $view_type v_b; statement ok drop $view_type v_c; +statement ok +drop $view_type v_d; + statement ok drop $view_type v_a_b; +statement ok +drop $view_type v_a_c; + +statement ok +drop $view_type v_a_d; + statement ok drop $view_type v_b_c; statement ok -drop $view_type v_a_c; +drop $view_type v_b_d; + +statement ok +drop $view_type v_c_d; statement ok drop $view_type v_a_b_c; +statement ok +drop $view_type v_a_b_c_d; + statement ok drop table t; diff --git a/e2e_test/s3/fs_source_batch.py b/e2e_test/s3/fs_source_batch.py new file mode 100644 index 0000000000000..d606be36f37f0 --- /dev/null +++ b/e2e_test/s3/fs_source_batch.py @@ -0,0 +1,155 @@ +import os +import sys +import csv +import json +import random +import psycopg2 + +from time import sleep +from io import StringIO +from minio import Minio +from functools import partial + +def gen_data(file_num, item_num_per_file): + assert item_num_per_file % 2 == 0, \ + f'item_num_per_file should be even to ensure sum(mark) == 0: {item_num_per_file}' + return [ + [{ + 'id': file_id * item_num_per_file + item_id, + 'name': f'{file_id}_{item_id}', + 'sex': item_id % 2, + 'mark': (-1) ** (item_id % 2), + } for item_id in range(item_num_per_file)] + for file_id in range(file_num) + ] + +def format_json(data): + return [ + '\n'.join([json.dumps(item) for item in file]) + for file in data + ] + +def format_csv(data, with_header): + csv_files = [] + + for file_data in data: + ostream = StringIO() + writer = csv.DictWriter(ostream, fieldnames=file_data[0].keys()) + if with_header: + writer.writeheader() + for item_data in file_data: + writer.writerow(item_data) + csv_files.append(ostream.getvalue()) + return csv_files + +def do_test(config, file_num, item_num_per_file, prefix, fmt): + conn = psycopg2.connect( + host="localhost", + port="4566", + user="root", + database="dev" + ) + + # Open a cursor to execute SQL statements + cur = conn.cursor() + + def _source(): + return f's3_test_{fmt}' + + def _encode(): + if fmt == 'json': + return 'JSON' + else: + return f"CSV (delimiter = ',', without_header = {str('without' in fmt).lower()})" + + # Execute a SELECT statement + cur.execute(f'''CREATE SOURCE {_source()}( + id int, + name TEXT, + sex int, + mark int, + ) WITH ( + connector = 's3_v2', + match_pattern = '{prefix}*.{fmt}', + s3.region_name = '{config['S3_REGION']}', + s3.bucket_name = '{config['S3_BUCKET']}', + s3.credentials.access = '{config['S3_ACCESS_KEY']}', + s3.credentials.secret = '{config['S3_SECRET_KEY']}', + s3.endpoint_url = 'https://{config['S3_ENDPOINT']}' + ) FORMAT PLAIN ENCODE {_encode()};''') + + total_rows = file_num * item_num_per_file + MAX_RETRIES = 40 + for retry_no in range(MAX_RETRIES): + cur.execute(f'select count(*) from {_source()}') + result = cur.fetchone() + if result[0] == total_rows: + break + print(f"[retry {retry_no}] Now got {result[0]} rows in source, {total_rows} expected, wait 30s") + sleep(30) + + stmt = f'select count(*), sum(id), sum(sex), sum(mark) from {_source()}' + print(f'Execute {stmt}') + cur.execute(stmt) + result = cur.fetchone() + + print('Got:', result) + + def _assert_eq(field, got, expect): + assert got == expect, f'{field} assertion failed: got {got}, expect {expect}.' + + _assert_eq('count(*)', result[0], total_rows) + _assert_eq('sum(id)', result[1], (total_rows - 1) * total_rows / 2) + _assert_eq('sum(sex)', result[2], total_rows / 2) + _assert_eq('sum(mark)', result[3], 0) + + print('Test pass') + + cur.execute(f'drop source {_source()}') + cur.close() + conn.close() + + +if __name__ == "__main__": + FILE_NUM = 4001 + ITEM_NUM_PER_FILE = 2 + data = gen_data(FILE_NUM, ITEM_NUM_PER_FILE) + + fmt = sys.argv[1] + FORMATTER = { + 'json': format_json, + 'csv_with_header': partial(format_csv, with_header=True), + 'csv_without_header': partial(format_csv, with_header=False), + } + assert fmt in FORMATTER, f"Unsupported format: {fmt}" + formatted_files = FORMATTER[fmt](data) + + config = json.loads(os.environ["S3_SOURCE_TEST_CONF"]) + client = Minio( + config["S3_ENDPOINT"], + access_key=config["S3_ACCESS_KEY"], + secret_key=config["S3_SECRET_KEY"], + secure=True, + ) + run_id = str(random.randint(1000, 9999)) + _local = lambda idx: f'data_{idx}.{fmt}' + _s3 = lambda idx: f"{run_id}_data_{idx}.{fmt}" + + # put s3 files + for idx, file_str in enumerate(formatted_files): + with open(_local(idx), "w") as f: + f.write(file_str) + os.fsync(f.fileno()) + + client.fput_object( + config["S3_BUCKET"], + _s3(idx), + _local(idx) + ) + + # do test + do_test(config, FILE_NUM, ITEM_NUM_PER_FILE, run_id, fmt) + + # clean up s3 files + for idx, _ in enumerate(formatted_files): + client.remove_object(config["S3_BUCKET"], _s3(idx)) diff --git a/e2e_test/schema_registry/alter_sr.slt b/e2e_test/schema_registry/alter_sr.slt new file mode 100644 index 0000000000000..dc05f81fc1362 --- /dev/null +++ b/e2e_test/schema_registry/alter_sr.slt @@ -0,0 +1,74 @@ +# Before running this test, seed data into kafka: +# python3 e2e_test/schema_registry/pb.py + +statement ok +CREATE SOURCE src_user WITH ( + connector = 'kafka', + topic = 'sr_pb_test', + properties.bootstrap.server = 'message_queue:29092', + scan.startup.mode = 'earliest' +) +FORMAT PLAIN ENCODE PROTOBUF( + schema.registry = 'http://message_queue:8081', + message = 'test.User' +); + +statement ok +CREATE MATERIALIZED VIEW mv_user AS SELECT * FROM src_user; + +# Changing type is not allowed +statement error Feature is not yet implemented: this altering statement will drop columns, which is not supported yet: \(city: character varying\) +ALTER SOURCE src_user FORMAT PLAIN ENCODE PROTOBUF( + schema.registry = 'http://message_queue:8081', + message = 'test.UserWithNewType' +); + +# Changing format/encode is not allowed +statement error Feature is not yet implemented: the original definition is FORMAT Plain ENCODE Protobuf, and altering them is not supported yet +ALTER SOURCE src_user FORMAT NATIVE ENCODE PROTOBUF( + schema.registry = 'http://message_queue:8081', + message = 'test.User' +); + +statement ok +ALTER SOURCE src_user FORMAT PLAIN ENCODE PROTOBUF( + schema.registry = 'http://message_queue:8081', + message = 'test.UserWithMoreFields' +); + +# Dropping columns is not allowed +statement error Feature is not yet implemented: this altering statement will drop columns, which is not supported yet: \(age: integer\) +ALTER SOURCE src_user FORMAT PLAIN ENCODE PROTOBUF( + schema.registry = 'http://message_queue:8081', + message = 'test.User' +); + +statement ok +CREATE MATERIALIZED VIEW mv_more_fields AS SELECT * FROM src_user; + +system ok +python3 e2e_test/schema_registry/pb.py "message_queue:29092" "http://message_queue:8081" "sr_pb_test" 5 user_with_more_fields + +sleep 10s + +query I +SELECT COUNT(*) FROM mv_user; +---- +25 + +statement error +SELECT SUM(age) FROM mv_user; + +query III +SELECT COUNT(*), MAX(age), MIN(age) FROM mv_more_fields; +---- +25 4 0 + +statement ok +DROP MATERIALIZED VIEW mv_user; + +statement ok +DROP MATERIALIZED VIEW mv_more_fields; + +statement ok +DROP SOURCE src_user; diff --git a/e2e_test/schema_registry/pb.py b/e2e_test/schema_registry/pb.py index e83fd1d36f8a5..7ca15222e149d 100644 --- a/e2e_test/schema_registry/pb.py +++ b/e2e_test/schema_registry/pb.py @@ -25,18 +25,38 @@ def get_user(i): sc=SourceContext(file_name="source/context_{:03}.proto".format(i)), ) +def get_user_with_more_fields(i): + return user_pb2.UserWithMoreFields( + id=i, + name="User_{}".format(i), + address="Address_{}".format(i), + city="City_{}".format(i), + gender=user_pb2.MALE if i % 2 == 0 else user_pb2.FEMALE, + sc=SourceContext(file_name="source/context_{:03}.proto".format(i)), + age=i, + ) + +def get_user_with_new_type(i): + return user_pb2.UserWithNewType( + id=i, + name="User_{}".format(i), + address="Address_{}".format(i), + city=i, + gender=user_pb2.MALE if i % 2 == 0 else user_pb2.FEMALE, + sc=SourceContext(file_name="source/context_{:03}.proto".format(i)), + ) -def send_to_kafka(producer_conf, schema_registry_conf, topic, num_records): +def send_to_kafka(producer_conf, schema_registry_conf, topic, num_records, get_user_fn, pb_message): schema_registry_client = SchemaRegistryClient(schema_registry_conf) serializer = ProtobufSerializer( - user_pb2.User, + pb_message, schema_registry_client, {"use.deprecated.format": False, 'skip.known.types': True}, ) producer = Producer(producer_conf) for i in range(num_records): - user = get_user(i) + user = get_user_fn(i) producer.produce( topic=topic, @@ -49,20 +69,29 @@ def send_to_kafka(producer_conf, schema_registry_conf, topic, num_records): if __name__ == "__main__": - if len(sys.argv) < 4: - print("pb.py ") + if len(sys.argv) < 5: + print("pb.py ") exit(1) broker_list = sys.argv[1] schema_registry_url = sys.argv[2] topic = sys.argv[3] num_records = int(sys.argv[4]) + pb_message = sys.argv[5] + + all_pb_messages = { + 'user': (get_user, user_pb2.User), + 'user_with_more_fields': (get_user_with_more_fields, user_pb2.UserWithMoreFields), + 'user_with_new_type': (get_user_with_new_type, user_pb2.UserWithNewType), + } + + assert pb_message in all_pb_messages, f'pb_message must be one of {list(all_pb_messages.keys())}' schema_registry_conf = {"url": schema_registry_url} producer_conf = {"bootstrap.servers": broker_list} try: - send_to_kafka(producer_conf, schema_registry_conf, topic, num_records) + send_to_kafka(producer_conf, schema_registry_conf, topic, num_records, *all_pb_messages[pb_message]) except Exception as e: print("Send Protobuf data to schema registry and kafka failed {}", e) exit(1) diff --git a/e2e_test/schema_registry/pb.slt b/e2e_test/schema_registry/pb.slt index fb40759d34ada..d9c0edca1b21c 100644 --- a/e2e_test/schema_registry/pb.slt +++ b/e2e_test/schema_registry/pb.slt @@ -1,5 +1,5 @@ # Before running this test, seed data into kafka: -# python3 e2e_test/schema_registry/pb.py +# python3 e2e_test/schema_registry/pb.py # Create a table. statement ok diff --git a/e2e_test/schema_registry/protobuf/user.proto b/e2e_test/schema_registry/protobuf/user.proto index e6c5f109bbd76..bbff4b97bac9c 100644 --- a/e2e_test/schema_registry/protobuf/user.proto +++ b/e2e_test/schema_registry/protobuf/user.proto @@ -17,3 +17,22 @@ enum Gender { MALE = 0; FEMALE = 1; } + +message UserWithMoreFields { + int32 id = 1; + string name = 2; + string address = 3; + string city = 4; + Gender gender = 5; + google.protobuf.SourceContext sc = 6; + int32 age = 7; // new field here +} + +message UserWithNewType { + int32 id = 1; + string name = 2; + string address = 3; + int32 city = 4; // change the type from string to int32 + Gender gender = 5; + google.protobuf.SourceContext sc = 6; +} diff --git a/e2e_test/schema_registry/protobuf/user_pb2.py b/e2e_test/schema_registry/protobuf/user_pb2.py index b87f3a5ea1d81..bd7b61e646fb1 100644 --- a/e2e_test/schema_registry/protobuf/user_pb2.py +++ b/e2e_test/schema_registry/protobuf/user_pb2.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- # Generated by the protocol buffer compiler. DO NOT EDIT! # source: user.proto -# Protobuf Python Version: 4.25.0 +# Protobuf Python Version: 4.25.1 """Generated protocol buffer code.""" from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool @@ -15,15 +15,19 @@ from google.protobuf import source_context_pb2 as google_dot_protobuf_dot_source__context__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nuser.proto\x12\x04test\x1a$google/protobuf/source_context.proto\"\x89\x01\n\x04User\x12\n\n\x02id\x18\x01 \x01(\x05\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x01(\t\x12\x0c\n\x04\x63ity\x18\x04 \x01(\t\x12\x1c\n\x06gender\x18\x05 \x01(\x0e\x32\x0c.test.Gender\x12*\n\x02sc\x18\x06 \x01(\x0b\x32\x1e.google.protobuf.SourceContext*\x1e\n\x06Gender\x12\x08\n\x04MALE\x10\x00\x12\n\n\x06\x46\x45MALE\x10\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\nuser.proto\x12\x04test\x1a$google/protobuf/source_context.proto\"\x89\x01\n\x04User\x12\n\n\x02id\x18\x01 \x01(\x05\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x01(\t\x12\x0c\n\x04\x63ity\x18\x04 \x01(\t\x12\x1c\n\x06gender\x18\x05 \x01(\x0e\x32\x0c.test.Gender\x12*\n\x02sc\x18\x06 \x01(\x0b\x32\x1e.google.protobuf.SourceContext\"\xa4\x01\n\x12UserWithMoreFields\x12\n\n\x02id\x18\x01 \x01(\x05\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x01(\t\x12\x0c\n\x04\x63ity\x18\x04 \x01(\t\x12\x1c\n\x06gender\x18\x05 \x01(\x0e\x32\x0c.test.Gender\x12*\n\x02sc\x18\x06 \x01(\x0b\x32\x1e.google.protobuf.SourceContext\x12\x0b\n\x03\x61ge\x18\x07 \x01(\x05\"\x94\x01\n\x0fUserWithNewType\x12\n\n\x02id\x18\x01 \x01(\x05\x12\x0c\n\x04name\x18\x02 \x01(\t\x12\x0f\n\x07\x61\x64\x64ress\x18\x03 \x01(\t\x12\x0c\n\x04\x63ity\x18\x04 \x01(\x05\x12\x1c\n\x06gender\x18\x05 \x01(\x0e\x32\x0c.test.Gender\x12*\n\x02sc\x18\x06 \x01(\x0b\x32\x1e.google.protobuf.SourceContext*\x1e\n\x06Gender\x12\x08\n\x04MALE\x10\x00\x12\n\n\x06\x46\x45MALE\x10\x01\x62\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'user_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None - _globals['_GENDER']._serialized_start=198 - _globals['_GENDER']._serialized_end=228 + _globals['_GENDER']._serialized_start=516 + _globals['_GENDER']._serialized_end=546 _globals['_USER']._serialized_start=59 _globals['_USER']._serialized_end=196 + _globals['_USERWITHMOREFIELDS']._serialized_start=199 + _globals['_USERWITHMOREFIELDS']._serialized_end=363 + _globals['_USERWITHNEWTYPE']._serialized_start=366 + _globals['_USERWITHNEWTYPE']._serialized_end=514 # @@protoc_insertion_point(module_scope) diff --git a/e2e_test/sink/cassandra_sink.slt b/e2e_test/sink/cassandra_sink.slt new file mode 100644 index 0000000000000..7091e8da70783 --- /dev/null +++ b/e2e_test/sink/cassandra_sink.slt @@ -0,0 +1,33 @@ +statement ok +CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamptz, v9 boolean); + +statement ok +CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; + +statement ok +CREATE SINK s6 +FROM + mv6 WITH ( + connector = 'cassandra', + type = 'append-only', + force_append_only='true', + cassandra.url = 'cassandra-server:9042', + cassandra.keyspace = 'demo', + cassandra.table = 'demo_bhv_table', + cassandra.datacenter = 'datacenter1', +); + +statement ok +INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01+00:00' , false); + +statement ok +FLUSH; + +statement ok +DROP SINK s6; + +statement ok +DROP MATERIALIZED VIEW mv6; + +statement ok +DROP TABLE t6; \ No newline at end of file diff --git a/e2e_test/sink/clickhouse_sink.slt b/e2e_test/sink/clickhouse_sink.slt index 909bdbfd6356b..9791f484326d7 100644 --- a/e2e_test/sink/clickhouse_sink.slt +++ b/e2e_test/sink/clickhouse_sink.slt @@ -1,11 +1,11 @@ statement ok -CREATE TABLE t6 (v1 int primary key, v2 bigint, v3 varchar); +CREATE TABLE t6 (v1 int primary key, v2 bigint, v3 varchar, v4 smallint); statement ok CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; statement ok -CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH ( +CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3, mv6.v4 as v4 from mv6 WITH ( connector = 'clickhouse', type = 'append-only', force_append_only='true', @@ -17,7 +17,7 @@ CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH ); statement ok -INSERT INTO t6 VALUES (1, 50, '1-50'), (2, 2, '2-2'), (3, 2, '3-2'), (5, 2, '5-2'), (8, 2, '8-2'), (13, 2, '13-2'), (21, 2, '21-2'); +INSERT INTO t6 VALUES (1, 50, '1-50', 1), (2, 2, '2-2', 2), (3, 2, '3-2', 1), (5, 2, '5-2', 2), (8, 2, '8-2', 1), (13, 2, '13-2', 2), (21, 2, '21-2', 1); statement ok FLUSH; diff --git a/e2e_test/sink/doris_sink.slt b/e2e_test/sink/doris_sink.slt new file mode 100644 index 0000000000000..2c552bbb26143 --- /dev/null +++ b/e2e_test/sink/doris_sink.slt @@ -0,0 +1,34 @@ +statement ok +CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamp, v9 boolean); + +statement ok +CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; + +statement ok +CREATE SINK s6 +FROM + mv6 WITH ( + connector = 'doris', + type = 'append-only', + doris.url = 'http://doris-fe-server:8030', + doris.user = 'users', + doris.password = '123456', + doris.database = 'demo', + doris.table='demo_bhv_table', + force_append_only='true' +); + +statement ok +INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01' , false); + +statement ok +FLUSH; + +statement ok +DROP SINK s6; + +statement ok +DROP MATERIALIZED VIEW mv6; + +statement ok +DROP TABLE t6; \ No newline at end of file diff --git a/e2e_test/sink/iceberg_sink.slt b/e2e_test/sink/iceberg_sink.slt index 9e7bef8d12239..d8efe836b4995 100644 --- a/e2e_test/sink/iceberg_sink.slt +++ b/e2e_test/sink/iceberg_sink.slt @@ -14,9 +14,10 @@ CREATE SINK s6 AS select mv6.v1 as v1, mv6.v2 as v2, mv6.v3 as v3 from mv6 WITH s3.access.key = 'hummockadmin', s3.secret.key = 'hummockadmin', s3.region = 'us-east-1', + catalog.name = 'demo', catalog.type = 'storage', - database.name='demo', - table.name='demo_db.demo_table' + database.name='demo_db', + table.name='demo_table' ); statement ok @@ -25,12 +26,16 @@ INSERT INTO t6 VALUES (1, 2, '1-2'), (2, 2, '2-2'), (3, 2, '3-2'), (5, 2, '5-2') statement ok FLUSH; +sleep 5s + statement ok INSERT INTO t6 VALUES (1, 50, '1-50'); statement ok FLUSH; +sleep 5s + statement ok DROP SINK s6; diff --git a/e2e_test/sink/kafka/avro.slt b/e2e_test/sink/kafka/avro.slt index 45ecf302f0ddd..d9fa53bc589ac 100644 --- a/e2e_test/sink/kafka/avro.slt +++ b/e2e_test/sink/kafka/avro.slt @@ -1,5 +1,5 @@ statement ok -create table from_kafka ( primary key (some_key) ) +create table from_kafka ( *, gen_i32_field int as int32_field + 2, primary key (some_key) ) include key as some_key with ( connector = 'kafka', @@ -52,6 +52,7 @@ select float_field, double_field, int32_field, + gen_i32_field, int64_field, record_field, array_field, @@ -61,10 +62,10 @@ select time_micros_field, time_millis_field from from_kafka order by string_field; ---- -t Rising \x6130 3.5 4.25 22 23 NULL {{NULL,3},NULL,{7,NULL,2}} 2006-01-02 22:04:05+00:00 NULL NULL 12:34:56.123456 NULL -f Wave \x5a4446 1.5 NULL 11 12 (,foo) NULL NULL 2006-01-02 22:04:05+00:00 2021-04-01 NULL 23:45:16.654 +t Rising \x6130 3.5 4.25 22 24 23 NULL {{NULL,3},NULL,{7,NULL,2}} 2006-01-02 22:04:05+00:00 NULL NULL 12:34:56.123456 NULL +f Wave \x5a4446 1.5 NULL 11 13 12 (,foo) NULL NULL 2006-01-02 22:04:05+00:00 2021-04-01 NULL 23:45:16.654 -statement error SchemaFetchError +statement error test-rw-sink-upsert-avro-err-key create sink sink_err from into_kafka with ( connector = 'kafka', topic = 'test-rw-sink-upsert-avro-err', diff --git a/e2e_test/sink/redis_sink.slt b/e2e_test/sink/redis_sink.slt new file mode 100644 index 0000000000000..7475a80ae696e --- /dev/null +++ b/e2e_test/sink/redis_sink.slt @@ -0,0 +1,41 @@ +statement ok +CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamptz, v9 boolean); + +statement ok +CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; + +statement ok +CREATE SINK s61 +FROM + mv6 WITH ( + primary_key = 'v1', + connector = 'redis', + redis.url= 'redis://redis-server:6379/', +)FORMAT PLAIN ENCODE JSON(force_append_only='true'); + +statement ok +CREATE SINK s62 +FROM + mv6 WITH ( + primary_key = 'v1', + connector = 'redis', + redis.url= 'redis://redis-server:6379/', +)FORMAT PLAIN ENCODE TEMPLATE(force_append_only='true', key_format = 'V1:{v1}', value_format = 'V2:{v2},V3:{v3}'); + +statement ok +INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01+00:00' , false); + +statement ok +FLUSH; + +statement ok +DROP SINK s61; + +statement ok +DROP SINK s62; + +statement ok +DROP MATERIALIZED VIEW mv6; + +statement ok +DROP TABLE t6; \ No newline at end of file diff --git a/e2e_test/sink/sink_into_table/basic.slt b/e2e_test/sink/sink_into_table/basic.slt index 1bc5a47907077..890087e207fd0 100644 --- a/e2e_test/sink/sink_into_table/basic.slt +++ b/e2e_test/sink/sink_into_table/basic.slt @@ -362,6 +362,35 @@ drop table t_b; statement ok drop table t_c; +# cycle check (with materialize view) + +statement ok +create table t_a(v int primary key); + +statement ok +create materialized view m_a as select v from t_a; + +statement ok +create table t_b(v int primary key); + +statement ok +create sink s_a into t_b as select v from m_a; + +statement error Creating such a sink will result in circular dependency +create sink s_b into t_a as select v from t_b; + +statement ok +drop sink s_a; + +statement ok +drop table t_b; + +statement ok +drop materialized view m_a; + +statement ok +drop table t_a; + # multi sinks statement ok diff --git a/e2e_test/sink/sink_into_table/rename.slt b/e2e_test/sink/sink_into_table/rename.slt index 19b24c9c20b4a..f518349c01c27 100644 --- a/e2e_test/sink/sink_into_table/rename.slt +++ b/e2e_test/sink/sink_into_table/rename.slt @@ -19,7 +19,7 @@ ALTER SINK s RENAME TO s1; query TT SELECT name, definition from rw_sinks; ---- -s1 CREATE SINK s1 INTO target AS SELECT * FROM source WITH (type = 'append-only') +s1 CREATE SINK s1 INTO target AS SELECT * FROM source WITH (type = 'append-only') statement ok ALTER TABLE source RENAME TO src; @@ -36,7 +36,7 @@ tar CREATE TABLE tar (v INT) query TT SELECT name, definition from rw_sinks; ---- -s1 CREATE SINK s1 INTO target AS SELECT * FROM src AS source WITH (type = 'append-only') +s1 CREATE SINK s1 INTO tar AS SELECT * FROM src AS source WITH (type = 'append-only') statement ok drop sink s1; diff --git a/e2e_test/sink/starrocks_sink.slt b/e2e_test/sink/starrocks_sink.slt new file mode 100644 index 0000000000000..a1ee1b0ffe039 --- /dev/null +++ b/e2e_test/sink/starrocks_sink.slt @@ -0,0 +1,36 @@ +statement ok +CREATE TABLE t6 (v1 int primary key, v2 smallint, v3 bigint, v4 real, v5 float, v6 varchar, v7 date, v8 timestamp, v9 boolean, v10 jsonb); + +statement ok +CREATE MATERIALIZED VIEW mv6 AS SELECT * FROM t6; + +statement ok +CREATE SINK s6 +FROM + mv6 WITH ( + connector = 'starrocks', + type = 'upsert', + starrocks.host = 'starrocks-fe-server', + starrocks.mysqlport = '9030', + starrocks.httpport = '8030', + starrocks.user = 'users', + starrocks.password = '123456', + starrocks.database = 'demo', + starrocks.table = 'demo_bhv_table', + primary_key = 'v1' +); + +statement ok +INSERT INTO t6 VALUES (1, 1, 1, 1.1, 1.2, 'test', '2013-01-01', '2013-01-01 01:01:01' , false, '{"v101":100}'); + +statement ok +FLUSH; + +statement ok +DROP SINK s6; + +statement ok +DROP MATERIALIZED VIEW mv6; + +statement ok +DROP TABLE t6; \ No newline at end of file diff --git a/e2e_test/source/basic/alter/kafka.slt b/e2e_test/source/basic/alter/kafka.slt index 7b355f6407e52..45b0cdd860f3e 100644 --- a/e2e_test/source/basic/alter/kafka.slt +++ b/e2e_test/source/basic/alter/kafka.slt @@ -57,6 +57,10 @@ alter source s1 add column v2 varchar; statement ok alter source s2 add column v4 int; +# alter table with conn +statement error try to use ALTER TABLE instead +alter source t add column v0 int; + statement ok create materialized view mv3 as select * from s1; diff --git a/e2e_test/source/basic/ddl.slt b/e2e_test/source/basic/ddl.slt index f959ce7fa49a7..402cf129b86ba 100644 --- a/e2e_test/source/basic/ddl.slt +++ b/e2e_test/source/basic/ddl.slt @@ -28,7 +28,8 @@ db error: ERROR: Failed to run the query Caused by these errors (recent errors listed first): 1: gRPC request to meta service failed: Internal error 2: failed to create source worker - 3: missing field `properties.bootstrap.server` + 3: failed to parse json + 4: missing field `properties.bootstrap.server` statement error @@ -134,7 +135,7 @@ create source s ( statement ok create materialized view mv_1 as select * from s -statement error other relation\(s\) depend on it +statement error Permission denied drop source s statement ok diff --git a/e2e_test/source/basic/inlcude_key_as.slt b/e2e_test/source/basic/inlcude_key_as.slt index 83cace5eae6f6..c971e1ac074d0 100644 --- a/e2e_test/source/basic/inlcude_key_as.slt +++ b/e2e_test/source/basic/inlcude_key_as.slt @@ -61,12 +61,32 @@ WITH ( topic = 'kafka_additional_columns') FORMAT PLAIN ENCODE JSON +# header with varchar type & non-exist header key +statement ok +create table additional_columns_1 (a int) +include key as key_col +include partition as partition_col +include offset as offset_col +include timestamp as timestamp_col +include header 'header1' as header_col_1 +include header 'header2' as header_col_2 +include header 'header2' varchar as header_col_3 +include header 'header3' as header_col_4 +WITH ( + connector = 'kafka', + properties.bootstrap.server = 'message_queue:29092', + topic = 'kafka_additional_columns') +FORMAT PLAIN ENCODE JSON + statement ok select * from upsert_students_default_key; statement ok select * from additional_columns; +statement ok +select * from additional_columns_1; + # Wait enough time to ensure SourceExecutor consumes all Kafka data. sleep 3s @@ -98,8 +118,16 @@ FROM additional_columns limit 1; ---- header1 \x7631 +query TTTT +select header_col_1, header_col_2, header_col_3, header_col_4 from additional_columns_1 limit 1 +---- +\x7631 \x7632 v2 NULL + statement ok drop table upsert_students_default_key statement ok drop table additional_columns + +statement ok +drop table additional_columns_1 diff --git a/e2e_test/source/basic/kafka.slt b/e2e_test/source/basic/kafka.slt index ab7616084db5f..08c84947846a7 100644 --- a/e2e_test/source/basic/kafka.slt +++ b/e2e_test/source/basic/kafka.slt @@ -163,7 +163,7 @@ create table s8_no_schema_field ( properties.bootstrap.server = 'message_queue:29092' ) FORMAT DEBEZIUM ENCODE JSON -statement ok +statement error without schema registry create table s9 with ( connector = 'kafka', topic = 'avro_bin', @@ -171,13 +171,21 @@ create table s9 with ( scan.startup.mode = 'earliest' ) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-simple-schema.avsc'); +statement ok +create table s9 with ( + connector = 'kafka', + topic = 'avro_bin', + properties.bootstrap.server = 'message_queue:29092', + scan.startup.mode = 'earliest' +) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-simple-schema.avsc', with_deprecated_file_header = true); + statement ok create table s10 with ( connector = 'kafka', topic = 'avro_c_bin', properties.bootstrap.server = 'message_queue:29092', scan.startup.mode = 'earliest' -) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-complex-schema.avsc'); +) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-complex-schema.avsc', with_deprecated_file_header = true); statement ok create table s11 with ( @@ -282,7 +290,7 @@ create source s18 with ( topic = 'avro_c_bin', properties.bootstrap.server = 'message_queue:29092', scan.startup.mode = 'earliest' -) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-complex-schema.avsc'); +) FORMAT PLAIN ENCODE AVRO (schema.location = 'file:///risingwave/avro-complex-schema.avsc', with_deprecated_file_header = true); # we cannot use confluent schema registry when connector is not kafka statement error @@ -293,8 +301,8 @@ with ( properties.bootstrap.server = 'message_queue:29092' ) FORMAT PLAIN ENCODE AVRO (schema.registry = 'http://127.0.0.1:8081'); -# we cannot create debezium source with generated column -statement error +# create debezium source with generated column +statement ok create table s20 ( id integer primary key, first_name varchar, @@ -478,6 +486,18 @@ create table source_with_rdkafka_props (v1 int, v2 varchar) with ( properties.queued.max.messages.kbytes = 65536 ) FORMAT PLAIN ENCODE JSON +statement ok +CREATE TABLE debezium_ignore_key ( + order_id int primary key +) with ( + connector = 'kafka', + topic = 'debezium_mess_key', + kafka.brokers = 'message_queue:29092', + kafka.scan.startup.mode = 'earliest') +FORMAT DEBEZIUM ENCODE JSON ( + ignore_key = 'true' +) + statement ok flush; @@ -680,10 +700,10 @@ select count(*) from s16 ---- 0 -statement error Feature is not yet implemented: Alter source with schema registry +statement error Not supported: alter source with schema registry alter source s18 add column v10 int; -statement error Feature is not yet implemented: Alter source with schema registry +statement error Not supported: alter source with schema registry alter source s17 add column v10 int; query III rowsort @@ -799,6 +819,11 @@ select * from s27 ---- (9.5,7,12) 12.5 1 An ice sculpture {cold,ice} +query I +select count(order_id) from debezium_ignore_key +---- +3 + statement ok drop materialized view source_mv1 @@ -847,6 +872,9 @@ drop source s17 statement ok drop source s18 +statement ok +drop table s20 + statement ok drop table s21 @@ -885,3 +913,6 @@ drop table dbz_ignore_case_json; statement ok drop table source_with_rdkafka_props; + +statement ok +drop table debezium_ignore_key; diff --git a/e2e_test/source/basic/old_row_format_syntax/ddl.slt b/e2e_test/source/basic/old_row_format_syntax/ddl.slt index 2ec7239e61d2c..d5c41d4ded878 100644 --- a/e2e_test/source/basic/old_row_format_syntax/ddl.slt +++ b/e2e_test/source/basic/old_row_format_syntax/ddl.slt @@ -90,7 +90,7 @@ create source s ( statement ok create materialized view mv_1 as select * from s -statement error other relation\(s\) depend on it +statement error Permission denied drop source s statement ok diff --git a/e2e_test/source/basic/old_row_format_syntax/kafka.slt b/e2e_test/source/basic/old_row_format_syntax/kafka.slt index 8e8c42a6e41f3..6c387aeb4523f 100644 --- a/e2e_test/source/basic/old_row_format_syntax/kafka.slt +++ b/e2e_test/source/basic/old_row_format_syntax/kafka.slt @@ -155,7 +155,7 @@ create table s8_no_schema_field ( properties.bootstrap.server = 'message_queue:29092' ) ROW FORMAT DEBEZIUM_JSON -statement ok +statement error without schema registry create table s9 with ( connector = 'kafka', topic = 'avro_bin', @@ -163,7 +163,7 @@ create table s9 with ( scan.startup.mode = 'earliest' ) row format avro row schema location 'file:///risingwave/avro-simple-schema.avsc' -statement ok +statement error without schema registry create table s10 with ( connector = 'kafka', topic = 'avro_c_bin', @@ -262,7 +262,7 @@ create source s17 with ( scan.startup.mode = 'earliest' ) row format protobuf message 'test.User' row schema location 'file:///risingwave/proto-complex-schema' -statement ok +statement error without schema registry create source s18 with ( connector = 'kafka', topic = 'avro_c_bin', @@ -280,8 +280,8 @@ with ( ) row format avro row schema location confluent schema registry 'http://127.0.0.1:8081' -# we cannot create debezium source with generated column -statement error +# create debezium source with generated column +statement ok create table s20 ( id integer primary key, first_name varchar, @@ -560,15 +560,15 @@ select id, first_name, last_name, email from s8_no_schema_field; 1004 Anne1 Kretchmar annek@noanswer.org 1005 add add2 add -query IITFFBTT -select id, sequence_id, name, score, avg_score, is_lasted, entrance_date, birthday, passed from s9; ----- -32 64 str_value 32 64 t 1970-01-01 1970-01-01 00:00:00+00:00 1 mon 1 day 00:00:01 - -query ITITT -select id, code, timestamp, xfas, contacts, sex from s10; ----- -100 abc 1473305798 {"(0,200,10.0.0.1)","(1,400,10.0.0.2)"} ("{1xxx,2xxx}","{1xxx,2xxx}") MALE +# query IITFFBTT +# select id, sequence_id, name, score, avg_score, is_lasted, entrance_date, birthday, passed from s9; +# ---- +# 32 64 str_value 32 64 t 1970-01-01 1970-01-01 00:00:00+00:00 1 mon 1 day 00:00:01 +# +# query ITITT +# select id, code, timestamp, xfas, contacts, sex from s10; +# ---- +# 100 abc 1473305798 {"(0,200,10.0.0.1)","(1,400,10.0.0.2)"} ("{1xxx,2xxx}","{1xxx,2xxx}") MALE query ITITT select id, code, timestamp, xfas, contacts, sex from s11; @@ -706,11 +706,11 @@ drop table s8 statement ok drop table s8_no_schema_field -statement ok -drop table s9 - -statement ok -drop table s10 +# statement ok +# drop table s9 +# +# statement ok +# drop table s10 statement ok drop table s11 @@ -733,8 +733,11 @@ drop table s16 statement ok drop source s17 +# statement ok +# drop source s18 + statement ok -drop source s18 +drop table s20 statement ok drop table s21 diff --git a/e2e_test/source/cdc/cdc.check_new_rows.slt b/e2e_test/source/cdc/cdc.check_new_rows.slt index a5e7c271b465e..5a97a21ba973a 100644 --- a/e2e_test/source/cdc/cdc.check_new_rows.slt +++ b/e2e_test/source/cdc/cdc.check_new_rows.slt @@ -68,9 +68,13 @@ SELECT * from orders_test_cnt 5 query ITT -SELECT * FROM products_test order by id limit 3 +SELECT * FROM rw.products_test order by id limit 3 ---- 101 RW Small 2-wheel scooter 102 RW 12V car battery 103 RW 12-pack of drill bits with sizes ranging from #40 to #3 +query TTTT +select order_date,customer_name,product_id,order_status from orders_no_backfill order by order_id; +---- +2022-12-01 15:08:22 Sam 110 0 \ No newline at end of file diff --git a/e2e_test/source/cdc/cdc.share_stream.slt b/e2e_test/source/cdc/cdc.share_stream.slt index babfb685a945c..7739d3f1ad6ea 100644 --- a/e2e_test/source/cdc/cdc.share_stream.slt +++ b/e2e_test/source/cdc/cdc.share_stream.slt @@ -31,7 +31,10 @@ create table products_test ( id INT, ) from mysql_mytest table 'products'; statement ok -create table products_test ( id INT, +create schema rw; + +statement ok +create table rw.products_test ( id INT, name STRING, description STRING, PRIMARY KEY (id) @@ -94,7 +97,7 @@ create table orders_test ( ) from mysql_mytest table 'mytest.orders'; statement ok -create materialized view products_test_cnt as select count(*) as cnt from products_test; +create materialized view products_test_cnt as select count(*) as cnt from rw.products_test; statement ok create materialized view orders_test_cnt as select count(*) as cnt from orders_test; @@ -102,8 +105,31 @@ create materialized view orders_test_cnt as select count(*) as cnt from orders_t system ok mysql --protocol=tcp -u root mytest -e "INSERT INTO products VALUES(default, 'Juice', '100ml Juice');" +system ok +mysql --protocol=tcp -u root mytest -e "FLUSH LOGS" + +# Should not contain records inserted before the table is created (e.g. 'Bob' inserted above) +statement ok +create table orders_no_backfill ( + order_id int, + order_date timestamp, + customer_name string, + price decimal, + product_id int, + order_status smallint, + PRIMARY KEY (order_id) +) with ( + snapshot = 'false' +) from mysql_mytest table 'mytest.orders'; + sleep 5s +# table without backfill should not contain historical records +query I +select count(*) from orders_no_backfill +---- +0 + # check ingestion results query I SELECT * from products_test_cnt @@ -116,7 +142,7 @@ SELECT * from orders_test_cnt 4 query ITT -SELECT * FROM products_test order by id limit 3 +SELECT * FROM rw.products_test order by id limit 3 ---- 101 scooter Small 2-wheel scooter 102 car battery 12V car battery @@ -179,6 +205,7 @@ CREATE TABLE IF NOT EXISTS postgres_all_types( c_timestamptz_array timestamptz[], c_interval_array interval[], c_jsonb_array jsonb[], + c_uuid varchar, PRIMARY KEY (c_boolean,c_bigint,c_date) ) from pg_source table 'public.postgres_all_types'; @@ -208,9 +235,9 @@ CREATE MATERIALIZED VIEW person_new_cnt AS SELECT COUNT(*) AS cnt FROM person_ne sleep 3s query TTTTTTT -SELECT c_boolean,c_date,c_time,c_timestamp,c_jsonb,c_smallint_array,c_timestamp_array FROM postgres_all_types where c_bigint=-9223372036854775807 +SELECT c_boolean,c_date,c_time,c_timestamp,c_jsonb,c_smallint_array,c_timestamp_array,c_uuid FROM postgres_all_types where c_bigint=-9223372036854775807 ---- -f 0001-01-01 00:00:00 0001-01-01 00:00:00 {} {-32767} {"0001-01-01 00:00:00"} +f 0001-01-01 00:00:00 0001-01-01 00:00:00 {} {-32767} {"0001-01-01 00:00:00"} bb488f9b-330d-4012-b849-12adeb49e57e # postgres streaming test diff --git a/e2e_test/source/cdc/mysql_init_data.sql b/e2e_test/source/cdc/mysql_init_data.sql index e954b74aaf500..d01165088eed1 100644 --- a/e2e_test/source/cdc/mysql_init_data.sql +++ b/e2e_test/source/cdc/mysql_init_data.sql @@ -1,4 +1,3 @@ --- USE `my@db`; INSERT INTO products VALUES (default,"scooter","Small 2-wheel scooter"), diff --git a/e2e_test/source/cdc/postgres_cdc.sql b/e2e_test/source/cdc/postgres_cdc.sql index 43dba14950b36..a4de0e447a0cc 100644 --- a/e2e_test/source/cdc/postgres_cdc.sql +++ b/e2e_test/source/cdc/postgres_cdc.sql @@ -67,7 +67,8 @@ CREATE TABLE IF NOT EXISTS postgres_all_types( c_timestamptz_array timestamptz[], c_interval_array interval[], c_jsonb_array jsonb[], + c_uuid uuid, PRIMARY KEY (c_boolean,c_bigint,c_date) ); -INSERT INTO postgres_all_types VALUES ( False, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'::timestamp, '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::real[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::time[], array[]::timestamp[], array[]::timestamptz[], array[]::interval[], array[]::jsonb[]); -INSERT INTO postgres_all_types VALUES ( False, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'::timestamp, '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['\x00'::bytea]::bytea[], array['0001-01-01'::date]::date[], array['00:00:00'::time]::time[], array['0001-01-01 00:00:00'::timestamp::timestamp]::timestamp[], array['0001-01-01 00:00:00'::timestamptz::timestamptz]::timestamptz[], array[interval '0 second'::interval]::interval[], array['{}'::jsonb]::jsonb[]); +INSERT INTO postgres_all_types VALUES ( False, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'::timestamp, '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::real[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::time[], array[]::timestamp[], array[]::timestamptz[], array[]::interval[], array[]::jsonb[], null); +INSERT INTO postgres_all_types VALUES ( False, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'::timestamp, '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['\x00'::bytea]::bytea[], array['0001-01-01'::date]::date[], array['00:00:00'::time]::time[], array['0001-01-01 00:00:00'::timestamp::timestamp]::timestamp[], array['0001-01-01 00:00:00'::timestamptz::timestamptz]::timestamptz[], array[interval '0 second'::interval]::interval[], array['{}'::jsonb]::jsonb[], 'bb488f9b-330d-4012-b849-12adeb49e57e'); diff --git a/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt b/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt index 461a5adf736ea..93cdb9882ebd4 100644 --- a/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt +++ b/e2e_test/source/nexmark_endless_sinks/nexmark_endless_part3.slt @@ -16,6 +16,9 @@ drop sink nexmark_q10; statement ok drop sink nexmark_q14; +statement ok +drop function count_char; + statement ok drop sink nexmark_q15; diff --git a/e2e_test/streaming/bug_fixes/issue_15075.slt b/e2e_test/streaming/bug_fixes/issue_15075.slt new file mode 100644 index 0000000000000..86d79e6f439a6 --- /dev/null +++ b/e2e_test/streaming/bug_fixes/issue_15075.slt @@ -0,0 +1,58 @@ +# https://github.com/risingwavelabs/risingwave/issues/15074 +# https://github.com/risingwavelabs/risingwave/issues/15075 + +statement ok +CREATE TABLE t1 ( + c1 varchar, + c2 varchar, + c3 bigint, + c4 numeric, + c5 timestamptz +) APPEND ONLY; + +statement ok +CREATE TABLE t2 ( + c6 timestamptz, + c7 integer, + c8 integer, + c9 integer, + c10 varchar, + c11 numeric, + c12 numeric, + c2 varchar +); + +statement ok +CREATE MATERIALIZED VIEW mv1 AS +SELECT + o.*, + b.c13, + b.c14 +FROM + t2 o + LEFT JOIN LATERAL ( + SELECT + sum(t.c3 * t.c4) filter ( + WHERE + (o.c11) * t.c4 <= (o.c11) * o.c12 + ) AS c13, + sum(t.c3) filter ( + WHERE + (o.c11) * t.c4 <= (o.c11) * o.c12 + ) AS c14 + FROM + t1 t + WHERE + t.c2 = o.c2 + AND t.c5 >= o.c6 + AND t.c5 :: date = o.c6 :: date + ) AS b ON TRUE; + +statement ok +drop materialized view mv1; + +statement ok +drop table t1; + +statement ok +drop table t2; diff --git a/e2e_test/streaming/bug_fixes/issue_15198.slt b/e2e_test/streaming/bug_fixes/issue_15198.slt new file mode 100644 index 0000000000000..a69aede18c2c9 --- /dev/null +++ b/e2e_test/streaming/bug_fixes/issue_15198.slt @@ -0,0 +1,23 @@ +# https://github.com/risingwavelabs/risingwave/issues/15198 + +statement ok +SET RW_IMPLICIT_FLUSH TO TRUE; + +statement ok +create materialized view "tumble_with_offset" +as ( + with + input as ( + select 1 as id, TO_TIMESTAMP('2024-01-01 01:30:02', 'YYYY-MM-DD HH24:MI:SS') as timestamps + ) + select * + from tumble(input, timestamps, interval '1 DAY', '+6 HOURS') +); + +query ITTT +select * from tumble_with_offset; +---- +1 2024-01-01 01:30:02+00:00 2023-12-31 06:00:00+00:00 2024-01-01 06:00:00+00:00 + +statement ok +drop materialized view tumble_with_offset; diff --git a/e2e_test/streaming/bug_fixes/issue_15302.slt b/e2e_test/streaming/bug_fixes/issue_15302.slt new file mode 100644 index 0000000000000..6d0bd01716cc6 --- /dev/null +++ b/e2e_test/streaming/bug_fixes/issue_15302.slt @@ -0,0 +1,38 @@ +# https://github.com/risingwavelabs/risingwave/issues/15302 + +statement ok +set RW_IMPLICIT_FLUSH = true; + +statement ok +create materialized view test_last_ingestion_time as ( +select 'table_a' as source, TO_TIMESTAMP('2024-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') as last_ingestion_time +union all +select 'table_b' as source, TO_TIMESTAMP('2024-01-01 00:00:01', 'YYYY-MM-DD HH24:MI:SS') as last_ingestion_time +); + +statement ok +create materialized view test_records as ( +select 1 as id, TO_TIMESTAMP('2024-01-01 00:00:00', 'YYYY-MM-DD HH24:MI:SS') as record_timestamp +union all +select 2 as id, TO_TIMESTAMP('2024-01-01 00:00:01', 'YYYY-MM-DD HH24:MI:SS') as record_timestamp +union all +select 3 as id, TO_TIMESTAMP('2024-01-01 00:00:02', 'YYYY-MM-DD HH24:MI:SS') as record_timestamp +); + +statement ok +create materialized view test_window as ( +with time_window as ( + select max(last_ingestion_time) as window_end + from test_last_ingestion_time +) +select + id + from test_records, time_window + where record_timestamp >= window_end +); + +query i +select * from test_window; +---- +2 +3 diff --git a/e2e_test/streaming/nexmark/drop_views.slt.part b/e2e_test/streaming/nexmark/drop_views.slt.part index a0084ebeff6ce..ccd5a522d5aa1 100644 --- a/e2e_test/streaming/nexmark/drop_views.slt.part +++ b/e2e_test/streaming/nexmark/drop_views.slt.part @@ -34,6 +34,9 @@ drop materialized view nexmark_q10; statement ok drop materialized view nexmark_q14; +statement ok +drop function count_char; + statement ok drop materialized view nexmark_q15; diff --git a/e2e_test/streaming/nexmark/q14.slt.part b/e2e_test/streaming/nexmark/q14.slt.part index c500c4d62c6f6..6e9de9bf85d8a 100644 --- a/e2e_test/streaming/nexmark/q14.slt.part +++ b/e2e_test/streaming/nexmark/q14.slt.part @@ -1,15 +1,15 @@ query IIRTT rowsort SELECT * FROM nexmark_q14; ---- -1000 1001 12696452.316 nightTime 2015-07-15 00:00:11.003 -1000 1001 1753269.004 nightTime 2015-07-15 00:00:14.004 -1000 1001 2196534.628 nightTime 2015-07-15 00:00:04.001 -1000 1001 26038438.152 nightTime 2015-07-15 00:00:13.003 -1000 1001 39654430.240 nightTime 2015-07-15 00:00:04.001 -1000 1001 9503126.184 nightTime 2015-07-15 00:00:22.005 -1000 1006 1301754.200 nightTime 2015-07-15 00:00:12.003 -1006 1001 19666916.800 nightTime 2015-07-15 00:00:16.004 -1006 1001 3099951.044 nightTime 2015-07-15 00:00:16.004 -1008 1002 26289625.456 nightTime 2015-07-15 00:00:04.001 -1012 1001 3067656.208 nightTime 2015-07-15 00:00:20.005 -1012 1001 3299152.624 nightTime 2015-07-15 00:00:20.005 +1000 1001 12696452.316 nightTime 2015-07-15 00:00:11.003 1 +1000 1001 1753269.004 nightTime 2015-07-15 00:00:14.004 0 +1000 1001 2196534.628 nightTime 2015-07-15 00:00:04.001 3 +1000 1001 26038438.152 nightTime 2015-07-15 00:00:13.003 3 +1000 1001 39654430.240 nightTime 2015-07-15 00:00:04.001 2 +1000 1001 9503126.184 nightTime 2015-07-15 00:00:22.005 1 +1000 1006 1301754.200 nightTime 2015-07-15 00:00:12.003 2 +1006 1001 19666916.800 nightTime 2015-07-15 00:00:16.004 4 +1006 1001 3099951.044 nightTime 2015-07-15 00:00:16.004 3 +1008 1002 26289625.456 nightTime 2015-07-15 00:00:04.001 4 +1012 1001 3067656.208 nightTime 2015-07-15 00:00:20.005 3 +1012 1001 3299152.624 nightTime 2015-07-15 00:00:20.005 3 diff --git a/e2e_test/streaming/nexmark/sinks/q14.slt.part b/e2e_test/streaming/nexmark/sinks/q14.slt.part index 77047c7f3c81c..cd9c18aad1bb9 100644 --- a/e2e_test/streaming/nexmark/sinks/q14.slt.part +++ b/e2e_test/streaming/nexmark/sinks/q14.slt.part @@ -1,3 +1,14 @@ +statement ok +CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$ + var count = 0; + for (var cc of s) { + if (cc === c) { + count++; + } + } + return count; +$$; + statement ok CREATE SINK nexmark_q14 AS SELECT @@ -15,11 +26,8 @@ SELECT THEN 'nightTime' ELSE 'otherTime' END AS bidTimeType, - date_time - -- extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java - -- count_char(extra, 'c') AS c_counts + date_time, + count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000 WITH ( connector = 'blackhole', type = 'append-only', force_append_only = 'true'); diff --git a/e2e_test/streaming/nexmark/views/q14.slt.part b/e2e_test/streaming/nexmark/views/q14.slt.part index 1182afb1e1989..7f4029c8defe6 100644 --- a/e2e_test/streaming/nexmark/views/q14.slt.part +++ b/e2e_test/streaming/nexmark/views/q14.slt.part @@ -1,3 +1,14 @@ +statement ok +CREATE FUNCTION count_char(s varchar, c varchar) RETURNS int LANGUAGE javascript AS $$ + var count = 0; + for (var cc of s) { + if (cc === c) { + count++; + } + } + return count; +$$; + statement ok CREATE MATERIALIZED VIEW nexmark_q14 AS SELECT @@ -15,10 +26,7 @@ SELECT THEN 'nightTime' ELSE 'otherTime' END AS bidTimeType, - date_time - -- extra - -- TODO: count_char is an UDF, add it back when we support similar functionality. - -- https://github.com/nexmark/nexmark/blob/master/nexmark-flink/src/main/java/com/github/nexmark/flink/udf/CountChar.java - -- count_char(extra, 'c') AS c_counts + date_time, + count_char(extra, 'c') AS c_counts FROM bid WHERE 0.908 * price > 1000000 AND 0.908 * price < 50000000; diff --git a/e2e_test/streaming/rate_limit.slt b/e2e_test/streaming/rate_limit/basic.slt similarity index 100% rename from e2e_test/streaming/rate_limit.slt rename to e2e_test/streaming/rate_limit/basic.slt diff --git a/e2e_test/streaming/rate_limit/upstream_amplification.slt b/e2e_test/streaming/rate_limit/upstream_amplification.slt new file mode 100644 index 0000000000000..71be801a78fc2 --- /dev/null +++ b/e2e_test/streaming/rate_limit/upstream_amplification.slt @@ -0,0 +1,44 @@ +# This test will test that barrier latency does not spike +# when there's rate limit on source. +# The upstream side should backpressure the source reader, +# but still allow barriers to flow through. + +statement ok +SET STREAMING_PARALLELISM=2; + +statement ok +SET STREAMING_RATE_LIMIT=1; + +statement ok +CREATE TABLE source_table (i1 int) +WITH ( + connector = 'datagen', + fields.i1.start = '1', + fields.i1.end = '5', + datagen.rows.per.second = '10000' +) FORMAT PLAIN ENCODE JSON; + +statement ok +CREATE SINK sink AS + SELECT x.i1 as i1 FROM source_table x + JOIN source_table s1 ON x.i1 = s1.i1 + JOIN source_table s2 ON x.i1 = s2.i1 + JOIN source_table s3 ON x.i1 = s3.i1 + WITH (connector = 'blackhole'); + +# The following sequence of FLUSH should be fast, since barrier should be able to bypass sink. +# Otherwise, these FLUSH will take a long time to complete, and trigger timeout. +statement ok +flush; + +statement ok +flush; + +statement ok +flush; + +statement ok +drop sink sink; + +statement ok +drop table source_table; \ No newline at end of file diff --git a/e2e_test/streaming/temporal_join/issue_15257.slt b/e2e_test/streaming/temporal_join/issue_15257.slt new file mode 100644 index 0000000000000..b376e8c68620e --- /dev/null +++ b/e2e_test/streaming/temporal_join/issue_15257.slt @@ -0,0 +1,31 @@ +statement ok +SET RW_IMPLICIT_FLUSH TO true; + +statement ok +create table stream(id1 int, a1 int, b1 int) APPEND ONLY; + +statement ok +create table version(id2 int, a2 int, b2 int, primary key (id2)); + +statement ok +create materialized view v as select id1, a1, id2, a2 from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on id1 = id2 and a1 > a2 + +statement ok +insert into version values(1, 11, 111); + +statement ok +insert into stream values(1, 10, 111); + +query IIII rowsort +select * from v; +---- +1 10 NULL NULL + +statement ok +drop materialized view v; + +statement ok +drop table stream; + +statement ok +drop table version; diff --git a/e2e_test/streaming/temporal_join/temporal_join_multiple_rows.slt b/e2e_test/streaming/temporal_join/temporal_join_multiple_rows.slt new file mode 100644 index 0000000000000..6cdeb901c39cf --- /dev/null +++ b/e2e_test/streaming/temporal_join/temporal_join_multiple_rows.slt @@ -0,0 +1,109 @@ +# The suite tests the cases that multiple rows are matched. + +statement ok +SET RW_IMPLICIT_FLUSH TO true; + +statement ok +create table stream(a1 int, b1 int) APPEND ONLY; + +statement ok +create table version(a2 int, b2 int); + +statement ok +create index idx on version (a2); + +statement ok +create materialized view v as +select a1, a2 from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on a1 = a2; + +statement ok +insert into stream values + (1,1) +,(2,1) +; + + +statement ok +insert into version values + (1,1) +,(1,2) +,(1,3) +; + +statement ok +insert into stream values + (1,1) +,(2,1) +; + +query II rowsort +select a1, a2 from v; +---- +1 1 +1 1 +1 1 +1 NULL +2 NULL +2 NULL + +statement ok +drop materialized view v; + +statement ok +drop table stream; + +statement ok +drop table version; + +# Test non equal conditions + +statement ok +create table stream(a1 int, b1 int) APPEND ONLY; + +statement ok +create table version(a2 int, b2 int); + +statement ok +create index idx on version (a2); + +statement ok +create materialized view v as +select a1, a2, b2 +from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() + on a1 = a2 and b1 > b2; + +statement ok +insert into version values + (1,1) +,(1,2) +,(1,3) +; + +statement ok +insert into stream values + (1,0) +,(1,3) +,(1,6) +,(2,1) +; + + +query III rowsort +select a1, a2, b2 from v; +---- +1 1 1 +1 1 1 +1 1 2 +1 1 2 +1 1 3 +1 NULL NULL +2 NULL NULL + +statement ok +drop materialized view v; + +statement ok +drop table stream; + +statement ok +drop table version; diff --git a/e2e_test/streaming/temporal_join/temporal_join_non_loopup_cond.slt b/e2e_test/streaming/temporal_join/temporal_join_non_loopup_cond.slt new file mode 100644 index 0000000000000..1a29d83615ec3 --- /dev/null +++ b/e2e_test/streaming/temporal_join/temporal_join_non_loopup_cond.slt @@ -0,0 +1,80 @@ +statement ok +SET RW_IMPLICIT_FLUSH TO true; + +statement ok +create table stream(id1 int, a1 int, b1 int) APPEND ONLY; + +statement ok +create table version(id2 int, a2 int, b2 int, primary key (id2)); + +statement ok +create materialized view v as select id1, a1, id2, a2 from stream left join version FOR SYSTEM_TIME AS OF PROCTIME() on id1 = id2 and a1 > a2; + +statement ok +insert into stream values(1, 11, 111); + +statement ok +insert into version values(1, 12, 111); + +statement ok +insert into stream values(1, 13, 111); + +statement ok +delete from version; + +query IIII rowsort +select * from v; +---- +1 11 NULL NULL +1 13 1 12 + +statement ok +insert into version values(2, 22, 222); + +statement ok +insert into stream values(2, 23, 222); + +query IIII rowsort +select * from v; +---- +1 11 NULL NULL +1 13 1 12 +2 23 2 22 + +statement ok +drop materialized view v; + +statement ok +drop table stream; + +statement ok +drop table version; + +statement ok +create table stream(id1 int, a1 int, b1 int) APPEND ONLY; + +statement ok +create table version(id2 int, a2 int, b2 int, primary key (id2)); + +statement ok +create materialized view v as select id1, a1, id2, a2 from stream join version FOR SYSTEM_TIME AS OF PROCTIME() on id1 = id2 and a1 > a2; + +statement ok +insert into version values (1, 12, 111); + +statement ok +insert into stream values (1, 11, 111), (1, 13, 111); + +query IIII rowsort +select * from v; +---- +1 13 1 12 + +statement ok +drop materialized view v; + +statement ok +drop table stream; + +statement ok +drop table version; diff --git a/e2e_test/streaming/temporal_join/temporal_join_watermark.slt b/e2e_test/streaming/temporal_join/temporal_join_watermark.slt index 9479ca6727299..c6c64e5774692 100644 --- a/e2e_test/streaming/temporal_join/temporal_join_watermark.slt +++ b/e2e_test/streaming/temporal_join/temporal_join_watermark.slt @@ -68,6 +68,3 @@ drop table stream cascade; statement ok drop table version cascade; - - - diff --git a/e2e_test/ttl/ttl.slt b/e2e_test/ttl/ttl.slt new file mode 100644 index 0000000000000..8361e19365edb --- /dev/null +++ b/e2e_test/ttl/ttl.slt @@ -0,0 +1,38 @@ +statement error +create table t(v int) with (retention_seconds = 5); + +statement ok +create table t(v int) APPEND ONLY with (retention_seconds = 5); + +statement ok +create index i on t(v); + +statement ok +insert into t values(1); + +statement ok +flush; + +query I +select * from t; +---- +1 + +query I +select * from i; +---- +1 + +statement ok +select pg_sleep(10); + +query I +select * from t; +---- + +query I +select * from i; +---- + +statement ok +drop table t; \ No newline at end of file diff --git a/e2e_test/udf/always_retry_python.slt b/e2e_test/udf/always_retry_python.slt new file mode 100644 index 0000000000000..64477a7df6be9 --- /dev/null +++ b/e2e_test/udf/always_retry_python.slt @@ -0,0 +1,75 @@ +system ok +python3 e2e_test/udf/test.py & + +# wait for server to start +sleep 10s + +statement ok +CREATE FUNCTION sleep_always_retry(INT) RETURNS INT AS 'sleep' USING LINK 'http://localhost:8815' WITH ( always_retry_on_network_error = true ); + +statement ok +CREATE FUNCTION sleep_no_retry(INT) RETURNS INT AS 'sleep' USING LINK 'http://localhost:8815'; + +# Create a table with 30 records +statement ok +CREATE TABLE t (v1 int); + +statement ok +INSERT INTO t select 0 from generate_series(1, 30); + +statement ok +flush; + +statement ok +SET STREAMING_RATE_LIMIT=1; + +statement ok +SET BACKGROUND_DDL=true; + +statement ok +CREATE MATERIALIZED VIEW mv_no_retry AS SELECT sleep_no_retry(v1) as s1 from t; + +# Create a Materialized View +statement ok +CREATE MATERIALIZED VIEW mv_always_retry AS SELECT sleep_always_retry(v1) as s1 from t; + +# Immediately kill the server, sleep for 1minute. +system ok +pkill -9 -i python && sleep 60 + +# Restart the server +system ok +python3 e2e_test/udf/test.py & + +# Wait for materialized view to be complete +statement ok +wait; + +query I +SELECT count(*) FROM mv_always_retry where s1 is NULL; +---- +0 + +query B +SELECT count(*) > 0 FROM mv_no_retry where s1 is NULL; +---- +t + +statement ok +SET STREAMING_RATE_LIMIT=0; + +statement ok +SET BACKGROUND_DDL=false; + +# close the server +system ok +pkill -i python + +statement ok +DROP FUNCTION sleep_always_retry; + +statement ok +DROP FUNCTION sleep_no_retry; + +statement ok +DROP TABLE t CASCADE; \ No newline at end of file diff --git a/e2e_test/udf/udf.slt b/e2e_test/udf/external_udf.slt similarity index 100% rename from e2e_test/udf/udf.slt rename to e2e_test/udf/external_udf.slt diff --git a/e2e_test/udf/js_udf.slt b/e2e_test/udf/js_udf.slt new file mode 100644 index 0000000000000..260fd991f648f --- /dev/null +++ b/e2e_test/udf/js_udf.slt @@ -0,0 +1,154 @@ +statement ok +create function int_42() returns int language javascript as $$ + return 42; +$$; + +query I +select int_42(); +---- +42 + +statement ok +drop function int_42; + + +statement ok +create function gcd(a int, b int) returns int language javascript as $$ + // required before we support `RETURNS NULL ON NULL INPUT` + if(a == null || b == null) { + return null; + } + while (b != 0) { + let t = b; + b = a % b; + a = t; + } + return a; +$$; + +query I +select gcd(25, 15); +---- +5 + +statement ok +drop function gcd; + + +statement ok +create function decimal_add(a decimal, b decimal) returns decimal language javascript as $$ + return a + b; +$$; + +query R +select decimal_add(1.11, 2.22); +---- +3.33 + +statement ok +drop function decimal_add; + + +statement ok +create function to_string(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb) returns varchar language javascript as $$ + return a.toString() + b.toString() + c.toString() + d.toString() + e.toString() + f.toString() + g.toString() + h.toString() + i.toString() + JSON.stringify(j); +$$; + +query T +select to_string(false, 1::smallint, 2, 3, 4.5, 6.7, 8.9, 'abc', '\x010203', '{"key": 1}'); +---- +false1234.56.78.9abc1,2,3{"key":1} + +statement ok +drop function to_string; + + +# show data types in javascript +statement ok +create function js_typeof(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb) returns jsonb language javascript as $$ + return { + boolean: typeof a, + smallint: typeof b, + int: typeof c, + bigint: typeof d, + real: typeof e, + float: typeof f, + decimal: typeof g, + varchar: typeof h, + bytea: typeof i, + jsonb: typeof j, + }; +$$; + +query T +select js_typeof(false, 1::smallint, 2, 3, 4.5, 6.7, 8.9, 'abc', '\x010203', '{"key": 1}'); +---- +{"bigint": "number", "boolean": "boolean", "bytea": "object", "decimal": "bigdecimal", "float": "number", "int": "number", "jsonb": "object", "real": "number", "smallint": "number", "varchar": "string"} + +statement ok +drop function js_typeof; + + +statement ok +create function return_all(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb, s struct) +returns struct> +language javascript as $$ + return {a,b,c,d,e,f,g,h,i,j,s}; +$$; + +query T +select (return_all( + true, + 1 ::smallint, + 1, + 1, + 1, + 1, + 12345678901234567890.12345678, + 'string', + 'bytes', + '{"key":1}', + row(1, 2)::struct +)).*; +---- +t 1 1 1 1 1 12345678901234567890.12345678 string \x6279746573 {"key": 1} (1,2) + +statement ok +drop function return_all; + + +statement ok +create function series(n int) returns table (x int) language javascript as $$ + for(let i = 0; i < n; i++) { + yield i; + } +$$; + +query I +select series(5); +---- +0 +1 +2 +3 +4 + +statement ok +drop function series; + + +statement ok +create function split(s varchar) returns table (word varchar, length int) language javascript as $$ + for(let word of s.split(' ')) { + yield { word: word, length: word.length }; + } +$$; + +query IT +select * from split('rising wave'); +---- +rising 6 +wave 4 + +statement ok +drop function split; diff --git a/e2e_test/udf/python_udf.slt b/e2e_test/udf/python_udf.slt new file mode 100644 index 0000000000000..b8253c96de4dc --- /dev/null +++ b/e2e_test/udf/python_udf.slt @@ -0,0 +1,195 @@ +statement ok +create function int_42() returns int language python as $$ +def int_42(): + return 42; +$$; + +query I +select int_42(); +---- +42 + +statement ok +drop function int_42; + + +statement ok +create function gcd(a int, b int) returns int language python as $$ +def gcd(a, b): + while b != 0: + a, b = b, a % b + return a +$$; + +query I +select gcd(25, 15); +---- +5 + +statement ok +drop function gcd; + + +statement ok +create function decimal_add(a decimal, b decimal) returns decimal language python as $$ +def decimal_add(a, b): + return a + b +$$; + +query R +select decimal_add(1.11, 2.22); +---- +3.33 + +statement ok +drop function decimal_add; + + +statement ok +create function to_string(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb) returns varchar language python as $$ +import json +def to_string(a, b, c, d, e, f, g, h, i, j): + return str(a) + str(b) + str(c) + str(d) + str(e) + str(f) + str(g) + str(h) + str(i) + json.dumps(j) +$$; + +query T +select to_string(false, 1::smallint, 2, 3, 4.5, 6.7, 8.9, 'abc', '\x010203', '{"key": 1}'); +---- +False1234.56.78.9abcb'\x01\x02\x03'{"key": 1} + +statement ok +drop function to_string; + + +# show data types in python +statement ok +create function py_typeof(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb) returns jsonb language python as $$ +def py_typeof(a, b, c, d, e, f, g, h, i, j): + return { + "boolean": type(a).__name__, + "smallint": type(b).__name__, + "int": type(c).__name__, + "bigint": type(d).__name__, + "real": type(e).__name__, + "float": type(f).__name__, + "decimal": type(g).__name__, + "varchar": type(h).__name__, + "bytea": type(i).__name__, + "jsonb": type(j).__name__, + }; +$$; + +query T +select py_typeof(false, 1::smallint, 2, 3, 4.5, 6.7, 8.9, 'abc', '\x010203', '{"key": 1}'); +---- +{"bigint": "int", "boolean": "bool", "bytea": "bytes", "decimal": "Decimal", "float": "float", "int": "int", "jsonb": "dict", "real": "float", "smallint": "int", "varchar": "str"} + +statement ok +drop function py_typeof; + + +statement ok +create function return_all(a boolean, b smallint, c int, d bigint, e real, f float, g decimal, h varchar, i bytea, j jsonb, s struct) +returns struct> +language python as $$ +class Ret: + def __init__(self, a, b, c, d, e, f, g, h, i, j, s): + self.a = a + self.b = b + self.c = c + self.d = d + self.e = e + self.f = f + self.g = g + self.h = h + self.i = i + self.j = j + self.s = s +def return_all(a, b, c, d, e, f, g, h, i, j, s): + return Ret(a, b, c, d, e, f, g, h, i, j, s) +$$; + +query T +select (return_all( + true, + 1 ::smallint, + 1, + 1, + 1, + 1, + 12345678901234567890.12345678, + 'string', + 'bytes', + '{"key":1}', + row(1, 2)::struct +)).*; +---- +t 1 1 1 1 1 12345678901234567890.12345678 string \x6279746573 {"key": 1} (1,2) + +statement ok +drop function return_all; + + +statement ok +create function series(n int) returns table (x int) language python as $$ +def series(n): + for i in range(n): + yield i +$$; + +query I +select series(5); +---- +0 +1 +2 +3 +4 + +statement ok +drop function series; + + +statement ok +create function split(s varchar) returns table (word varchar, length int) language python as $$ +class Ret: + def __init__(self, word, length): + self.word = word + self.length = length +def split(s): + for word in s.split(): + yield Ret(word, len(word)) +$$; + +query IT +select * from split('rising wave'); +---- +rising 6 +wave 4 + +statement ok +drop function split; + +statement ok +create function mismatched_arguments() returns int language python as $$ +def mismatched_arguments(x): + return x +$$; + +statement error missing 1 required positional argument: 'x' +select mismatched_arguments(); + +statement ok +drop function mismatched_arguments; + +statement ok +create function mismatched_return_type() returns int language python as $$ +def mismatched_return_type(): + return 1.0 +$$; + +statement error 'float' object cannot be interpreted as an integer +select mismatched_return_type(); + +statement ok +drop function mismatched_return_type; diff --git a/e2e_test/udf/sql_udf.slt b/e2e_test/udf/sql_udf.slt index 02fc23b2d7f02..866f27abd52ce 100644 --- a/e2e_test/udf/sql_udf.slt +++ b/e2e_test/udf/sql_udf.slt @@ -1,131 +1,177 @@ statement ok SET RW_IMPLICIT_FLUSH TO true; -# Create an anonymous function with double dollar as clause -statement ok -create function add(INT, INT) returns int language sql as $$select $1 + $2$$; - -# Create an anonymous function with single quote as clause -statement ok -create function sub(INT, INT) returns int language sql as 'select $1 - $2'; +############################################################# +# Basic tests for sql udf with [unnamed / named] parameters # +############################################################# -# Create an anonymous function that calls other pre-defined sql udfs +# Create a sql udf function with unnamed parameters with double dollar as clause statement ok -create function add_sub_binding() returns int language sql as 'select add(1, 1) + sub(2, 2)'; - -# Create an anonymous function that calls built-in functions -# Note that double dollar signs should be used otherwise the parsing will fail, as illutrates below -statement ok -create function call_regexp_replace() returns varchar language sql as $$select regexp_replace('💩💩💩💩💩foo🤔ï¸bar亲爱的😭bazè¿™ä¸æ˜¯çˆ±æƒ…â¤ï¸â€ðŸ”¥', 'baz(...)', '这是🥵', 'ic')$$; - -statement error Expected end of statement, found: 💩 -create function call_regexp_replace() returns varchar language sql as 'select regexp_replace('💩💩💩💩💩foo🤔ï¸bar亲爱的😭bazè¿™ä¸æ˜¯çˆ±æƒ…â¤ï¸â€ðŸ”¥', 'baz(...)', '这是🥵', 'ic')'; +create function add(INT, INT) returns int language sql as $$select $1 + $2$$; -# Create an anonymous function with return expression -statement ok -create function add_return(INT, INT) returns int language sql return $1 + $2; +query I +select add(1, -1); +---- +0 +# Create a sql udf function with unnamed parameters with single quote as clause statement ok -create function add_return_binding() returns int language sql return add_return(1, 1) + add_return(1, 1); +create function sub(INT, INT) returns int language sql as 'select $1 - $2'; -# Recursive definition can be accepted, but will be eventually rejected during runtime -statement ok -create function recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + recursive($1, $2)'; +query I +select sub(1, 1); +---- +0 -# Complex but error-prone definition, recursive & normal sql udfs interleaving +# Create a sql udf function with unamed parameters that calls other pre-defined sql udfs statement ok -create function recursive_non_recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + sub($1, $2)'; +create function add_sub_binding() returns int language sql as 'select add(1, 1) + sub(2, 2)'; -# Recursive corner case -statement ok -create function foo(INT) returns varchar language sql as $$select 'foo(INT)'$$; +query I +select add_sub_binding(); +---- +2 -# Create a wrapper function for `add` & `sub` -statement ok -create function add_sub_wrapper(INT, INT) returns int language sql as 'select add($1, $2) + sub($1, $2) + 114512'; +# Use them all together +query III +select add(1, -1), sub(1, 1), add_sub_binding(); +---- +0 0 2 -# Create a valid recursive function -# Please note we do NOT support actual running the recursive sql udf at present +# Create a sql udf with named parameters with single quote as clause statement ok -create function fib(INT) returns int - language sql as 'select case - when $1 = 0 then 0 - when $1 = 1 then 1 - when $1 = 2 then 1 - when $1 = 3 then 2 - else fib($1 - 1) + fib($1 - 2) - end;'; - -# The execution will eventually exceed the pre-defined max stack depth -statement error function fib calling stack depth limit exceeded -select fib(100); - -# Currently create a materialized view with a recursive sql udf will be rejected -statement error function fib calling stack depth limit exceeded -create materialized view foo_mv as select fib(100); +create function add_named(a INT, b INT) returns int language sql as 'select a + b'; -# Call the defined sql udf query I -select add(1, -1); +select add_named(1, -1); ---- 0 +# Create another sql udf with named parameters with double dollar as clause +statement ok +create function sub_named(a INT, b INT) returns int language sql as $$select a - b$$; + query I -select sub(1, 1); +select sub_named(1, 1); ---- 0 +# Mixed with named / unnamed parameters +statement ok +create function add_sub_mix(INT, a INT, INT) returns int language sql as 'select $1 - a + $3'; + query I -select add_sub_binding(); +select add_sub_mix(1, 2, 3); ---- 2 -query III -select add(1, -1), sub(1, 1), add_sub_binding(); +# Call sql udf with unnamed parameters inside sql udf with named parameters +statement ok +create function add_named_wrapper(a INT, b INT) returns int language sql as 'select add(a, b)'; + +query I +select add_named_wrapper(1, -1); ---- -0 0 2 +0 + +# Create a sql udf with unnamed parameters with return expression +statement ok +create function add_return(INT, INT) returns int language sql return $1 + $2; query I select add_return(1, 1); ---- 2 +statement ok +create function add_return_binding() returns int language sql return add_return(1, 1) + add_return(1, 1); + query I select add_return_binding(); ---- 4 +statement ok +create function print(INT) returns int language sql as 'select $1'; + query T -select call_regexp_replace(); +select print(114514); ---- -💩💩💩💩💩foo🤔ï¸bar亲爱的😭这是🥵爱情â¤ï¸â€ðŸ”¥ +114514 -query T -select foo(114514); +# Multiple type interleaving sql udf +statement ok +create function add_sub(INT, FLOAT, INT) returns float language sql as $$select -$1 + $2 - $3$$; + +query I +select add_sub(1, 5.1415926, 1); ---- -foo(INT) +3.1415926 -# Rejected deep calling stack -statement error function recursive calling stack depth limit exceeded -select recursive(1, 1); +query III +select add(1, -1), sub(1, 1), add_sub(1, 5.1415926, 1); +---- +0 0 3.1415926 -# Same as above -statement error function recursive calling stack depth limit exceeded -select recursive_non_recursive(1, 1); +# Complex types interleaving +statement ok +create function add_sub_types(INT, BIGINT, FLOAT, DECIMAL, REAL) returns double language sql as 'select $1 + $2 - $3 + $4 + $5'; + +query I +select add_sub_types(1, 1919810114514, 3.1415926, 1.123123, 101010.191919); +---- +1919810215523.1734494 + +statement ok +create function add_sub_return(INT, FLOAT, INT) returns float language sql return -$1 + $2 - $3; + +query I +select add_sub_return(1, 5.1415926, 1); +---- +3.1415926 + +# Create a wrapper function for `add` & `sub` +statement ok +create function add_sub_wrapper(INT, INT) returns int language sql as 'select add($1, $2) + sub($1, $2) + 114512'; query I select add_sub_wrapper(1, 1); ---- 114514 -# Create a mock table +########################################################## +# Basic sql udfs integrated with the use of mock tables # +# P.S. This is also a simulation of real world use cases # +########################################################## + statement ok create table t1 (c1 INT, c2 INT); -# Insert some data into the mock table +statement ok +create table t2 (c1 INT, c2 FLOAT, c3 INT); + +# Special table for named sql udf +statement ok +create table t3 (a INT, b INT); + statement ok insert into t1 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5); +statement ok +insert into t2 values (1, 3.14, 2), (2, 4.44, 5), (20, 10.30, 02); + +statement ok +insert into t3 values (1, 1), (2, 2), (3, 3), (4, 4), (5, 5); + +query I +select c1, c2, add_return(c1, c2) from t1 order by c1 asc; +---- +1 1 2 +2 2 4 +3 3 6 +4 4 8 +5 5 10 + query III select sub(c1, c2), c1, c2, add(c1, c2) from t1 order by c1 asc; ---- @@ -135,80 +181,373 @@ select sub(c1, c2), c1, c2, add(c1, c2) from t1 order by c1 asc; 0 4 4 8 0 5 5 10 +query IIIIII +select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub(c1, c2, c3) from t2 order by c1 asc; +---- +1 3.14 2 3 -1 0.14000000000000012 +2 4.44 5 7 -3 -2.5599999999999996 +20 10.3 2 22 18 -11.7 + +query IIIIII +select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub_return(c1, c2, c3) from t2 order by c1 asc; +---- +1 3.14 2 3 -1 0.14000000000000012 +2 4.44 5 7 -3 -2.5599999999999996 +20 10.3 2 22 18 -11.7 + query I -select c1, c2, add_return(c1, c2) from t1 order by c1 asc; +select add_named(a, b) from t3 order by a asc; ---- -1 1 2 -2 2 4 -3 3 6 -4 4 8 -5 5 10 +2 +4 +6 +8 +10 -# Recursive sql udf with normal table -statement error function fib calling stack depth limit exceeded -select fib(c1) from t1; +################################ +# Corner & Special cases tests # +################################ -# Recursive sql udf with materialized view -statement error function fib calling stack depth limit exceeded -create materialized view bar_mv as select fib(c1) from t1; +# Mixed parameter with calling inner sql udfs +statement ok +create function add_sub_mix_wrapper(INT, a INT, INT) returns int language sql as 'select add($1, a) + a + sub(a, $3)'; -# Invalid function body syntax -statement error Expected an expression:, found: EOF at the end -create function add_error(INT, INT) returns int language sql as $$select $1 + $2 +$$; +query I +select add_sub_mix_wrapper(1, 2, 3); +---- +4 -# Multiple type interleaving sql udf +# Named sql udf with corner case statement ok -create function add_sub(INT, FLOAT, INT) returns float language sql as $$select -$1 + $2 - $3$$; +create function corner_case(INT, a INT, INT) returns varchar language sql as $$select '$1 + a + $3'$$; -# Complex types interleaving +query T +select corner_case(1, 2, 3); +---- +$1 + a + $3 + +# Create a sql udf with unnamed parameters that calls built-in functions +# Note that double dollar signs should be used otherwise the parsing will fail, as illutrates below statement ok -create function add_sub_types(INT, BIGINT, FLOAT, DECIMAL, REAL) returns real language sql as 'select $1 + $2 - $3 + $4 + $5'; +create function call_regexp_replace() returns varchar language sql as $$select regexp_replace('💩💩💩💩💩foo🤔ï¸bar亲爱的😭bazè¿™ä¸æ˜¯çˆ±æƒ…â¤ï¸â€ðŸ”¥', 'baz(...)', '这是🥵', 'ic')$$; + +query T +select call_regexp_replace(); +---- +💩💩💩💩💩foo🤔ï¸bar亲爱的😭这是🥵爱情â¤ï¸â€ðŸ”¥ statement ok -create function add_sub_return(INT, FLOAT, INT) returns float language sql return -$1 + $2 - $3; +create function regexp_replace_wrapper(varchar) returns varchar language sql as $$select regexp_replace($1, 'baz(...)', '这是🥵', 'ic')$$; -query I -select add_sub(1, 5.1415926, 1); +query T +select regexp_replace_wrapper('💩💩💩💩💩foo🤔ï¸bar亲爱的😭bazè¿™ä¸æ˜¯çˆ±æƒ…â¤ï¸â€ðŸ”¥'); ---- -3.1415926 +💩💩💩💩💩foo🤔ï¸bar亲爱的😭这是🥵爱情â¤ï¸â€ðŸ”¥ -query I -select add_sub_return(1, 5.1415926, 1); +# Recursive corner case (i.e., valid definition should not be rejected) +statement ok +create function foo(INT) returns varchar language sql as $$select 'foo(INT)'$$; + +query T +select foo(114514); ---- -3.1415926 +foo(INT) + +# Adjust the input value of the calling function (i.e., `print` here) with the actual input parameter +statement ok +create function print_add_one(INT) returns int language sql as 'select print($1 + 1)'; + +statement ok +create function print_add_two(INT) returns int language sql as 'select print($1 + $1)'; query III -select add(1, -1), sub(1, 1), add_sub(1, 5.1415926, 1); +select print_add_one(1), print_add_one(114513), print_add_two(2); ---- -0 0 3.1415926 +2 114514 4 -query I -select add_sub_types(1, 1919810114514, 3.1415926, 1.123123, 101010.191919); +# Note: the valid example of `type_mismatch` in the below test section +statement ok +create function type_match(INT) returns varchar language sql as $$select '$1 + 114514 + $1'$$; + +query T +select type_match(114514); ---- -1919810215523.1734494 +$1 + 114514 + $1 -# Create another mock table -statement ok -create table t2 (c1 INT, c2 FLOAT, c3 INT); +################################################## +# Invalid definition tests when creating sql udf # +################################################## -statement ok -insert into t2 values (1, 3.14, 2), (2, 4.44, 5), (20, 10.30, 02); +# Named sql udf with invalid parameter in body definition +# Will be rejected at creation time +statement error +create function unknown_parameter(a INT) returns int language sql as 'select a + aa + a'; +---- +db error: ERROR: Failed to run the query -query IIIIII -select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub(c1, c2, c3) from t2 order by c1 asc; +Caused by: + Invalid input syntax: Failed to conduct semantic check +Bind error: [sql udf] failed to find named parameter aa +In SQL UDF definition: `select a + aa + a` + ^^ + + +# With unnamed parameter +statement error +create function unnamed_param_hint(INT, INT) returns int language sql as 'select $1 + $3 + $2'; ---- -1 3.14 2 3 -1 0.14000000000000012 -2 4.44 5 7 -3 -2.5599999999999996 -20 10.3 2 22 18 -11.7 +db error: ERROR: Failed to run the query -query IIIIII -select c1, c2, c3, add(c1, c3), sub(c1, c3), add_sub_return(c1, c2, c3) from t2 order by c1 asc; +Caused by: + Invalid input syntax: Failed to conduct semantic check +Bind error: [sql udf] failed to find unnamed parameter $3 +In SQL UDF definition: `select $1 + $3 + $2` + ^^ + + +# A mixture of both +statement error +create function mix_hint(INT, aa INT, INT) returns int language sql as 'select $1 + aa + a + $2'; ---- -1 3.14 2 3 -1 0.14000000000000012 -2 4.44 5 7 -3 -2.5599999999999996 -20 10.3 2 22 18 -11.7 +db error: ERROR: Failed to run the query + +Caused by: + Invalid input syntax: Failed to conduct semantic check +Bind error: [sql udf] failed to find named parameter a +In SQL UDF definition: `select $1 + aa + a + $2` + ^ + + +# Long invalid parameter +statement error +create function long_invalid_param(i_am_valid INT) returns int language sql as + 'select i_am_valid + _this_is_invalid_please_properly_mark_the_boundary_of_this_invalid_paramerter_ + i_am_valid'; +---- +db error: ERROR: Failed to run the query + +Caused by: + Invalid input syntax: Failed to conduct semantic check +Bind error: [sql udf] failed to find named parameter _this_is_invalid_please_properly_mark_the_boundary_of_this_invalid_paramerter_ +In SQL UDF definition: `select i_am_valid + _this_is_invalid_please_properly_mark_the_boundary_of_this_invalid_paramerter_ + i_am_valid` + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + +# sql udf with non-existent function +statement error +create function call_non_existent_function(INT) returns int language sql as 'select 1 + non_existent(1) + 1'; +---- +db error: ERROR: Failed to run the query + +Caused by: + Invalid input syntax: Failed to conduct semantic check +function non_existent(integer) does not exist +In SQL UDF definition: `select 1 + non_existent(1) + 1` + ^^^^^^^^^^^^ + + +# First display invalid parameter +# if not found, try displaying non-existent functions +# if still not found, display default error message without hint +statement error +create function param_func_mix(a INT, b INT) returns int language sql as 'select a + b + c + not_be_displayed(c)'; +---- +db error: ERROR: Failed to run the query + +Caused by: + Invalid input syntax: Failed to conduct semantic check +Bind error: [sql udf] failed to find named parameter c +In SQL UDF definition: `select a + b + c + not_be_displayed(c)` + ^ + + +statement error Expected end of statement, found: 💩 +create function call_regexp_replace() returns varchar language sql as 'select regexp_replace('💩💩💩💩💩foo🤔ï¸bar亲爱的😭bazè¿™ä¸æ˜¯çˆ±æƒ…â¤ï¸â€ðŸ”¥', 'baz(...)', '这是🥵', 'ic')'; + +# Recursive definition can NOT be accepted at present due to semantic check +statement error +create function recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + recursive($1, $2)'; +---- +db error: ERROR: Failed to run the query + +Caused by: + Invalid input syntax: Failed to conduct semantic check +function recursive(integer, integer) does not exist +In SQL UDF definition: `select recursive($1, $2) + recursive($1, $2)` + ^^^^^^^^^ + + +# Complex but error-prone definition, recursive & normal sql udfs interleaving +statement error Failed to conduct semantic check +create function recursive_non_recursive(INT, INT) returns int language sql as 'select recursive($1, $2) + sub($1, $2)'; + +# Create a valid recursive function +# Please note we do NOT support actual running the recursive sql udf at present +statement error Failed to conduct semantic check +create function fib(INT) returns int + language sql as 'select case + when $1 = 0 then 0 + when $1 = 1 then 1 + when $1 = 2 then 1 + when $1 = 3 then 2 + else fib($1 - 1) + fib($1 - 2) + end;'; + +# Calling a non-existent function +statement error Failed to conduct semantic check +create function non_exist(INT) returns int language sql as 'select yo(114514)'; + +# Try to create an sql udf with unnamed parameters whose return type mismatches with the sql body definition +statement error return type mismatch detected +create function type_mismatch(INT) returns varchar language sql as 'select $1 + 114514 + $1'; + +# Invalid function body syntax +statement error Expected an expression:, found: EOF at the end +create function add_error(INT, INT) returns int language sql as $$select $1 + $2 +$$; + +###################################################################### +# Not yet supported features 🤪 (and potential basic use case tests) # +###################################################################### + +# 1. Recursion Support for SQL UDF + +# The execution will eventually exceed the pre-defined max stack depth +# statement error function fib calling stack depth limit exceeded +# select fib(100); + +# Currently create a materialized view with a recursive sql udf will be rejected +# statement error function fib calling stack depth limit exceeded +# create materialized view foo_mv as select fib(100); + +# Rejected deep calling stack +# statement error function recursive calling stack depth limit exceeded +# select recursive(1, 1); + +# Same as above +# statement error function recursive calling stack depth limit exceeded +# select recursive_non_recursive(1, 1); + +# Recursive sql udf with normal table +# statement error function fib calling stack depth limit exceeded +# select fib(c1) from t1; + +# Recursive sql udf with materialized view +# statement error function fib calling stack depth limit exceeded +# create materialized view bar_mv as select fib(c1) from t1; + +# 2. Select from table inside SQL UDF (potential concerns: naming conflict) + +# statment ok +# create funciton from_table_single_row() returns int language sql as 'select c1 from t1'; + +# Do NOT need explicit `select ... from ...` +# query I +# select from_table_single_row(); +# ---- +# 1 + +# statement ok +# create function from_table_single_row_1() returns int language sql as 'select c1 from t1 order by c1 desc'; + +# Need explict `select ... from ...` +# query I +# select * from from_table_single_row_1(); +# ---- +# 5 + +# Should add parser support +# statement ok +# create function from_table_multiple_rows() returns setof int language sql as 'select c1 from t1'; + +# P.S. postgres shadows the `c1` parameter by the actual column `c1` to resolve the naming conflict +# statement ok +# create function from_table_conflict(c1 INT) returns int language sql as 'select c1 from t1'; + +# 3. Output multiple columns / expressions from a single SQL UDF + +# Parser support is needed +# statement ok +# create function out_parameters_without_name(out INT, out INT) language sql as 'select 1919810, 114514'; + +# query II +# select out_parameters_without_name(); +# ---- +# 1919810 114514 + +# query II +# select * out_parameters_without_name(); +# ---- +# 1919810 114514 + +# statement error non-existent column +# select a, b from out_parameters_without_name(); +# ---- +# 1919810 114514 + +# statement ok +# create function out_parameters_with_name(out a INT, out b INT) language sql as 'select 1919810, 114514'; + +# query II +# select out_parameters_with_name(); +# ---- +# 1919810 114514 + +# query II +# select a, b from out_parameters_with_name(); +# ---- +# 1919810 114514 + +# statement ok +# create function multiple_cols() returns setof record as 'select c1, c2, c3 from t2' language sql; + +# query III +# select * from multiple_cols() as t(c1 INT, c2 FLOAT, c3 INT); +# ---- +# corresponding results +# may need to order the sequence + +# 4. Polymorphic arguments + +# statement ok +# create function is_greater(anyelement, anyelement) returns boolean language sql as 'select $1 > $2'; + +# query I +# select is_greater(1, 2); +# ---- +# f + +# query I +# select is_greater(3.14, 2.95); +# ---- +# t + +# 5. Overloading functions + +# statement ok +# create function overload(c1 INT) returns int language sql as 'select c1'; + +# statement ok +# create function overload(c1 VARCHAR) returns varchar language sql as 'select c1'; + +# query I +# select overload(114514), overload('114514'); +# ---- +# 114514 114514 + +# statement error naming conflict with first overload +# create function overload(c1 INT, out VARCHAR) + +# This definition will cause ambiguity with the second overload during runtime if `c2` is not specified +# statement ok +# create function overload(c1 VARCHAR, c2 VARCHAR default '114514', out VARCHAR, out VARCHAR) language sql as 'select c1, c2'; + +# statement error can not choose a best candidate function, need explicit type cast +# query TT +# select overload('114514'); + +################################################## +# Clean up the funtions / mock tables at the end # +################################################## -# Drop the functions statement ok drop function add; @@ -237,23 +576,49 @@ statement ok drop function add_sub_wrapper; statement ok -drop function recursive; +drop function foo; statement ok -drop function foo; +drop function add_sub_types; statement ok -drop function recursive_non_recursive; +drop function print; statement ok -drop function add_sub_types; +drop function print_add_one; + +statement ok +drop function print_add_two; + +statement ok +drop function regexp_replace_wrapper; + +statement ok +drop function corner_case; + +statement ok +drop function add_named; statement ok -drop function fib; +drop function sub_named; + +statement ok +drop function add_sub_mix; + +statement ok +drop function add_named_wrapper; + +statement ok +drop function type_match; + +statement ok +drop function add_sub_mix_wrapper; -# Drop the mock table statement ok drop table t1; statement ok drop table t2; + +statement ok +drop table t3; \ No newline at end of file diff --git a/e2e_test/udf/wasm/Cargo.toml b/e2e_test/udf/wasm/Cargo.toml index 5e413a40e37b8..79d911279d63c 100644 --- a/e2e_test/udf/wasm/Cargo.toml +++ b/e2e_test/udf/wasm/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" crate-type = ["cdylib"] [dependencies] -arrow-udf = { git = "https://github.com/risingwavelabs/arrow-udf.git", rev = "f9a9e0d" } +arrow-udf = "0.1" genawaiter = "0.99" rust_decimal = "1" serde_json = "1" diff --git a/e2e_test/udf/wasm/src/lib.rs b/e2e_test/udf/wasm/src/lib.rs index 522dd471daf50..556022f9c6e1f 100644 --- a/e2e_test/udf/wasm/src/lib.rs +++ b/e2e_test/udf/wasm/src/lib.rs @@ -1,6 +1,11 @@ use arrow_udf::function; use rust_decimal::Decimal; +#[function("count_char(varchar, varchar) -> int")] +fn count_char(s: &str, c: &str) -> i32 { + s.matches(c).count() as i32 +} + #[function("int_42() -> int")] fn int_42() -> i32 { 42 diff --git a/e2e_test/udf/wasm_udf.slt b/e2e_test/udf/wasm_udf.slt index d182e8ff0f993..c2c97a029a1ad 100644 --- a/e2e_test/udf/wasm_udf.slt +++ b/e2e_test/udf/wasm_udf.slt @@ -91,3 +91,24 @@ drop function jsonb_access; statement ok drop function series; + +# inlined rust function +statement ok +create function gcd(int, int) returns int language rust as $$ + fn gcd(mut a: i32, mut b: i32) -> i32 { + while b != 0 { + let t = b; + b = a % b; + a = t; + } + a + } +$$; + +query I +select gcd(25, 15); +---- +5 + +statement ok +drop function gcd; diff --git a/grafana/common.py b/grafana/common.py index fbc8e71a3f098..9a47bd0c964a9 100644 --- a/grafana/common.py +++ b/grafana/common.py @@ -467,11 +467,12 @@ def timeseries_id(self, title, description, targets): **self.common_options, ) - def table_info(self, title, description, targets, excluded_columns): + def table_info(self, title, description, targets, columns): gridPos = self.layout.next_half_width_graph() - excludedByName = dict.fromkeys(excluded_columns, True) + column_indices = {column: index for index, column in enumerate(columns)} + excludeByName = dict.fromkeys(["Time", "Value"], True) transformations = [ - {"id": "organize", "options": {"excludeByName": excludedByName}} + {"id": "organize", "options": {"indexByName": column_indices, "excludeByName": excludeByName}} ] return Table( title=title, @@ -484,6 +485,7 @@ def table_info(self, title, description, targets, excluded_columns): transformations=transformations, ) + def sub_panel(self): return Panels(self.datasource) diff --git a/grafana/risingwave-dev-dashboard.dashboard.py b/grafana/risingwave-dev-dashboard.dashboard.py index 42cbafc536e6e..ff68308a938dc 100644 --- a/grafana/risingwave-dev-dashboard.dashboard.py +++ b/grafana/risingwave-dev-dashboard.dashboard.py @@ -24,22 +24,21 @@ def section_actor_info(outer_panels): panels = outer_panels.sub_panel() - excluded_cols = ["Time", "Value", "__name__", f"{COMPONENT_LABEL}", f"{NODE_LABEL}"] return [ outer_panels.row_collapsed( "Actor/Table Id Info", [ panels.table_info( - "Actor Id Info", - "Mapping from actor id to fragment id", - [panels.table_target(f"{metric('actor_info')}")], - excluded_cols, + "Actor Info", + "Information about actors", + [panels.table_target(f"group({metric('actor_info')}) by (actor_id, fragment_id, compute_node)")], + ["actor_id", "fragment_id", "compute_node"], ), panels.table_info( - "Materialized View Info", - "Mapping from materialized view table id to it's internal table ids", - [panels.table_target(f"{metric('table_info')}")], - excluded_cols, + "State Table Info", + "Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.", + [panels.table_target(f"group({metric('table_info')}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)")], + ["table_id", "table_name", "table_type", "materialized_view_id", "fragment_id", "compaction_group_id"], ), ], ) @@ -684,9 +683,8 @@ def section_streaming(outer_panels): "The figure shows the number of rows read by each source per second.", [ panels.target( - f"rate({metric('stream_source_output_rows_counts')}[$__rate_interval])", - "source={{source_name}} actor={{actor_id}} @ {{%s}}" - % NODE_LABEL, + f"sum(rate({metric('stream_source_output_rows_counts')}[$__rate_interval])) by (source_id, source_name, fragment_id)", + "{{source_id}} {{source_name}} (fragment {{fragment_id}})", ), ], ), @@ -697,7 +695,7 @@ def section_streaming(outer_panels): [ panels.target( f"rate({metric('source_partition_input_count')}[$__rate_interval])", - "actor={{actor_id}} source={{source_id}} partition={{partition}}", + "actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}", ) ], ), @@ -706,8 +704,8 @@ def section_streaming(outer_panels): "The figure shows the number of bytes read by each source per second.", [ panels.target( - f"(sum by (source_id)(rate({metric('source_partition_input_bytes')}[$__rate_interval])))/(1000*1000)", - "source={{source_id}}", + f"(sum by (source_id, source_name, fragment_id)(rate({metric('source_partition_input_bytes')}[$__rate_interval])))/(1000*1000)", + "{{source_id}} {{source_name}} (fragment {{fragment_id}})", ) ], ), @@ -718,20 +716,7 @@ def section_streaming(outer_panels): [ panels.target( f"(rate({metric('source_partition_input_bytes')}[$__rate_interval]))/(1000*1000)", - "actor={{actor_id}} source={{source_id}} partition={{partition}}", - ) - ], - ), - panels.timeseries_rowsps( - "Source Throughput(rows) per barrier", - "RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of " - "barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two " - "consecutive barriers.", - [ - panels.target( - f"rate({metric('stream_source_rows_per_barrier_counts')}[$__rate_interval])", - "actor={{actor_id}} source={{source_id}} @ {{%s}}" - % NODE_LABEL, + "actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}", ) ], ), @@ -833,6 +818,28 @@ def section_streaming(outer_panels): ), ], ), + panels.timeseries_rowsps( + "Arrangement Backfill Snapshot Read Throughput(rows)", + "Total number of rows that have been read from the backfill snapshot", + [ + panels.target( + f"rate({table_metric('stream_arrangement_backfill_snapshot_read_row_count')}[$__rate_interval])", + "table_id={{table_id}} actor={{actor_id}} @ {{%s}}" + % NODE_LABEL, + ), + ], + ), + panels.timeseries_rowsps( + "Arrangement Backfill Upstream Throughput(rows)", + "Total number of rows that have been output from the backfill upstream", + [ + panels.target( + f"rate({table_metric('stream_arrangement_backfill_upstream_output_row_count')}[$__rate_interval])", + "table_id={{table_id}} actor={{actor_id}} @ {{%s}}" + % NODE_LABEL, + ), + ], + ), panels.timeseries_count( "Barrier Number", "The number of barriers that have been ingested but not completely processed. This metric reflects the " @@ -991,6 +998,16 @@ def section_streaming_cdc(outer_panels): ), ], ), + panels.timeseries_count( + "CDC Source Errors", + "", + [ + panels.target( + f"sum({metric('cdc_source_error')}) by (connector_name, source_id, error_msg)", + "{{connector_name}}: {{error_msg}} ({{source_id}})", + ), + ], + ), ], ), ] @@ -1220,7 +1237,7 @@ def section_streaming_actors(outer_panels): "", [ panels.target( - f"sum(rate({metric('stream_join_actor_input_waiting_duration_ns')}[$__rate_interval]) / 1000000000) by (fragment_id)", + f"avg(rate({metric('stream_join_actor_input_waiting_duration_ns')}[$__rate_interval]) / 1000000000) by (fragment_id)", "fragment {{fragment_id}}", ), panels.target_hidden( @@ -1234,7 +1251,7 @@ def section_streaming_actors(outer_panels): "", [ panels.target( - f"sum(rate({metric('stream_join_match_duration_ns')}[$__rate_interval]) / 1000000000) by (fragment_id)", + f"avg(rate({metric('stream_join_match_duration_ns')}[$__rate_interval]) / 1000000000) by (fragment_id,side)", "fragment {{fragment_id}} {{side}}", ), panels.target_hidden( @@ -3648,6 +3665,26 @@ def section_memory_manager(outer_panels): ), ], ), + panels.timeseries_memory( + "The resident memory of jemalloc", + "", + [ + panels.target( + f"{metric('jemalloc_resident_bytes')}", + "", + ), + ], + ), + panels.timeseries_memory( + "The metadata memory of jemalloc", + "", + [ + panels.target( + f"{metric('jemalloc_metadata_bytes')}", + "", + ), + ], + ), panels.timeseries_memory( "The allocated memory of jvm", "", @@ -4032,12 +4069,12 @@ def section_udf(outer_panels): "udf_failure_count - {{%s}}" % NODE_LABEL, ), panels.target( - f"sum(rate({metric('udf_success_count')}[$__rate_interval])) by (link, name)", - "udf_success_count - {{link}} {{name}}", + f"sum(rate({metric('udf_success_count')}[$__rate_interval])) by (link, name, fragment_id)", + "udf_success_count - {{link}} {{name}} {{fragment_id}}", ), panels.target( - f"sum(rate({metric('udf_failure_count')}[$__rate_interval])) by (link, name)", - "udf_failure_count - {{link}} {{name}}", + f"sum(rate({metric('udf_failure_count')}[$__rate_interval])) by (link, name, fragment_id)", + "udf_failure_count - {{link}} {{name}} {{fragment_id}}", ), ], ), @@ -4046,8 +4083,8 @@ def section_udf(outer_panels): "", [ panels.target( - f"sum(irate({metric('udf_input_chunk_rows_sum')}[$__rate_interval])) by (link, name) / sum(irate({metric('udf_input_chunk_rows_count')}[$__rate_interval])) by (link, name)", - "udf_input_chunk_rows_avg - {{link}} {{name}}", + f"sum(irate({metric('udf_input_chunk_rows_sum')}[$__rate_interval])) by (link, name, fragment_id) / sum(irate({metric('udf_input_chunk_rows_count')}[$__rate_interval])) by (link, name, fragment_id)", + "udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}", ), ], ), @@ -4068,9 +4105,17 @@ def section_udf(outer_panels): "udf_latency_p99 - {{%s}}" % NODE_LABEL, ), panels.target( - f"sum(irate({metric('udf_latency_sum')}[$__rate_interval])) / sum(irate({metric('udf_latency_count')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL})", + f"sum(irate({metric('udf_latency_sum')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL}) / sum(irate({metric('udf_latency_count')}[$__rate_interval])) by ({COMPONENT_LABEL}, {NODE_LABEL})", "udf_latency_avg - {{%s}}" % NODE_LABEL, ), + panels.target( + f"histogram_quantile(0.99, sum(irate({metric('udf_latency_bucket')}[$__rate_interval])) by (le, link, name, fragment_id))", + "udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}", + ), + panels.target( + f"sum(irate({metric('udf_latency_sum')}[$__rate_interval])) by (link, name, fragment_id) / sum(irate({metric('udf_latency_count')}[$__rate_interval])) by (link, name, fragment_id)", + "udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}", + ), ], ), panels.timeseries_count( @@ -4082,8 +4127,8 @@ def section_udf(outer_panels): "udf_throughput_rows - {{%s}}" % NODE_LABEL, ), panels.target( - f"sum(rate({metric('udf_input_rows')}[$__rate_interval])) by (link, name)", - "udf_throughput_rows - {{link}} {{name}}", + f"sum(rate({metric('udf_input_rows')}[$__rate_interval])) by (link, name, fragment_id)", + "udf_throughput_rows - {{link}} {{name}} {{fragment_id}}", ), ], ), @@ -4096,8 +4141,8 @@ def section_udf(outer_panels): "udf_throughput_bytes - {{%s}}" % NODE_LABEL, ), panels.target( - f"sum(rate({metric('udf_input_bytes')}[$__rate_interval])) by (link, name) / (1024*1024)", - "udf_throughput_bytes - {{link}} {{name}}", + f"sum(rate({metric('udf_input_bytes')}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)", + "udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}", ), ], ), diff --git a/grafana/risingwave-dev-dashboard.json b/grafana/risingwave-dev-dashboard.json index 1db138e79999f..2fa9628478105 100644 --- a/grafana/risingwave-dev-dashboard.json +++ b/grafana/risingwave-dev-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"table_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RisingWave ingests barriers periodically to trigger computation and checkpoints. The frequency of barrier can be set by barrier_interval_ms. This metric shows how many rows are ingested between two consecutive barriers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_rows_per_barrier_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows) per barrier","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - fragment_id {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":35,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":39,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":65,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":67,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":82,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":84,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":85,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":87,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, executor_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{executor_id}})","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink by Connector","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":90,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":92,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":96,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":98,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":117,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":119,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":130,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":132,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":156,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":158,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":165,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":167,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":184,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":186,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":204,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":206,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":207,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":209,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":211,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":213,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":214,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":216,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":218,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":220,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":223,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":225,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":231,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":233,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":240,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":242,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":252,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":253,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":256,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":267,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":271,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":285,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":289,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":293,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":297,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":299,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":303,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dev Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (total) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cpu usage (avg per core) - {{job}} @ {{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"RW cluster can configure multiple meta nodes to achieve high availability. One is the leader and the rest are the followers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_addr}} @ {{role}}","metric":"","query":"sum(meta_num{job=~\"$job\",instance=~\"$node\"}) by (worker_addr,role)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Meta Cluster","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Cluster Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":9,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The rate of successful recovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery Successful Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of failed reocovery attempts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Failed recovery attempts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Time spent in a successful recovery attempt","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(recovery_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"recovery latency avg","metric":"","query":"sum by (le) (rate(recovery_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by (le) (rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Recovery latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Recovery","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":3},"height":null,"hideTimeOverride":false,"id":13,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(rows).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":15,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","query":"rate(source_partition_input_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Each query is executed in parallel with a user-defined parallelism. This figure shows the throughput of each parallelism. The throughput of all the parallelism added up is equal to Source Throughput(MB/s).","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor={{actor_id}} source={{source_id}} partition={{partition}} fragmend_id={{fragment_id}}","metric":"","query":"(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s) Per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Monitor each source upstream, 0 means the upstream is not normal, 1 means the source is ready.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":18,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Upstream Status","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Source Split Change Events frequency by source_id and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_split_change_event_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Split Change Events frequency(events/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Kafka Consumer Lag Size by source_id, partition and actor_id","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}}","metric":"","query":"source_kafka_high_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}} partition={{partition}} actor_id={{actor_id}}","metric":"","query":"source_latest_message_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kafka Consumer Lag Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}} - actor {{actor_id}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id, actor_id) * on(actor_id) group_left(sink_name) sink_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}} - fragment_id {{fragment_id}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, table_id) * on(fragment_id, table_id) group_left(table_name) table_info{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s) per Partition","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":27,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_arrangement_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_arrangement_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Arrangement Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_arrangement_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_arrangement_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Arrangement Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of barriers that have been ingested but not completely processed. This metric reflects the current level of congestion within the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all_barrier","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"in_flight_barrier","metric":"","query":"in_flight_barrier_nums{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The duration between the time point when the scheduled barrier needs to be sent and the time point when the barrier gets actually sent to all the compute nodes. Developers can thus detect any internal congestion.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_send_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_send_latency_avg","metric":"","query":"rate(meta_barrier_send_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_send_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Send Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":31,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_inflight_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_inflight_latency_avg","metric":"","query":"max(sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_inflight_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier In-Flight Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_p999 - {{instance}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_pmax - {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_barrier_sync_storage_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_sync_latency_avg - {{instance}}","metric":"","query":"sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, instance)(rate(stream_barrier_sync_storage_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Sync Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_barrier_wait_commit_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_wait_commit_avg","metric":"","query":"rate(meta_barrier_wait_commit_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_wait_commit_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Wait Commit Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of actors that have processed the earliest in-flight barriers per second. This metric helps users to detect potential congestion or stuck in the system.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"rate(stream_barrier_manager_progress{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Earliest In-Flight Barrier Progress","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":4},"height":null,"hideTimeOverride":false,"id":36,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the cdc backfill snapshot","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_snapshot_read_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Snapshot Read Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been output from the cdc backfill upstream","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":38,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_cdc_backfill_upstream_output_row_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Backfill Upstream Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p50 - {{table_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag p99 - {{table_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lag pmax - {{table_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(source_cdc_event_lag_duration_milliseconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_name))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Consume Lag Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{source_id}})","metric":"","query":"sum(cdc_source_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, source_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CDC Source Errors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming CDC","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":5},"height":null,"hideTimeOverride":false,"id":41,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_input_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":44,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}<-{{upstream_fragment_id}}","metric":"","query":"sum(rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, upstream_fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_in_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Input Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_actor_out_record_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Throughput (rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The operator-level memory usage statistics collected by each LRU cache","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} desc: {{desc}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (table_id, desc)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"stream_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Memory usage aggregated by materialized views","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Memory Usage of Materialized Views","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"temporal join cache miss, table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache hit count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_hit_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total cached count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_materialize_cache_total_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialize Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":50,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache lookup count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cache miss count - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_over_window_cache_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache lookup count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_lookup_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache left miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_left_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"partition range cache right miss count - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_over_window_range_cache_right_miss_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} fragment {{fragment_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize executor cache miss ratio - table {{table_id}} fragment {{fragment_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window cache miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache left miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_left_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Over window partition range cache right miss ratio - table {{table_id}} fragment {{fragment_id}} ","metric":"","query":"(sum(rate(stream_over_window_range_cache_right_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id) ) / (sum(rate(stream_over_window_range_cache_lookup_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p999 - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(0.999, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_barrier_align_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, wait_side, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} {{wait_side}} - {{job}}","metric":"","query":"sum by(le, fragment_id, wait_side, job)(rate(stream_join_barrier_align_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le,fragment_id,wait_side,job) (rate(stream_join_barrier_align_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Barrier Align","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":53,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"avg(rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"rate(stream_join_actor_input_waiting_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Input Blocking Time Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":54,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"avg(rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000) by (fragment_id,side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"rate(stream_join_match_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Actor Match Duration Per Second","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Multiple rows with distinct primary keys may have the same join key. This metric counts the number of join keys in the executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":55,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}} {{side}}","metric":"","query":"sum(stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (fragment_id, side)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}} {{side}}","metric":"","query":"stream_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of matched rows on the opposite side","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":56,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(stream_join_matched_join_keys_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, fragment_id, table_id, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - fragment {{fragment_id}} table_id {{table_id}} - {{job}}","metric":"","query":"sum by(le, job, actor_id, table_id) (rate(stream_join_matched_join_keys_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, fragment_id, table_id) (rate(stream_join_matched_join_keys_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Join Executor Matched Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":57,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level cache miss - table {{table_id}} actor {{actor_id}}}","metric":"","query":"rate(stream_agg_chunk_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"chunk-level total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_chunk_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Executor Cache Statistics For Each StreamChunk","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":58,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg distinct cached keys count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_distinct_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":59,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The total heap size of dirty (unflushed) groups in each hash aggregation executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":60,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"stream agg dirty groups heap size | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_agg_dirty_groups_heap_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregation Dirty Groups Heap Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in each top_n executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":61,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"group top_n appendonly cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_group_top_n_appendonly_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"TopN Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in temporal join executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":62,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal Join cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_temporal_join_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Temporal Join Cache Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in lookup executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":63,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lookup cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_lookup_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lookup Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of keys cached in over window executor's executor cache.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":64,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window cached count | table {{table_id}} actor {{actor_id}}","metric":"","query":"stream_over_window_cached_entry_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"over window partition range cache entry count | table {{table_id}} fragment {{fragment_id}}","metric":"","query":"sum(stream_over_window_range_cache_entry_count{job=~\"$job\",instance=~\"$node\"}) by (table_id, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Over Window Cached Keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When enabled, this metric shows the input throughput of each executor.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":65,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} fragment {{fragment_id}}","metric":"","query":"sum(rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_identity, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_identity}} actor {{actor_id}}","metric":"","query":"rate(stream_executor_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The actor-level memory usage statistics reported by TaskLocalAlloc. (Disabled by default)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":66,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}","metric":"","query":"sum(actor_memory_usage{job=~\"$job\",instance=~\"$node\"}) by (fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"actor {{actor_id}}","metric":"","query":"actor_memory_usage{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Memory Usage (TaskLocalAlloc)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":6},"height":null,"hideTimeOverride":false,"id":67,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":68,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_actor_execution_time{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Execution Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":69,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":70,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":8},"height":null,"hideTimeOverride":false,"id":71,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_fast_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_fast_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Fast Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":8},"height":null,"hideTimeOverride":false,"id":72,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":73,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":16},"height":null,"hideTimeOverride":false,"id":74,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_slow_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_slow_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Slow Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":16},"height":null,"hideTimeOverride":false,"id":75,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":76,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":24},"height":null,"hideTimeOverride":false,"id":77,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_poll_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_poll_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Poll Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":24},"height":null,"hideTimeOverride":false,"id":78,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":79,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":32},"height":null,"hideTimeOverride":false,"id":80,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_idle_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_idle_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Idle Avg Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":32},"height":null,"hideTimeOverride":false,"id":81,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Total Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":82,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":40},"height":null,"hideTimeOverride":false,"id":83,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{actor_id}}","metric":"","query":"rate(stream_actor_scheduled_duration{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(stream_actor_scheduled_cnt{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Tokio: Actor Scheduled Avg Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Actors (Tokio)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":7},"height":null,"hideTimeOverride":false,"id":84,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":85,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Send Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":86,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{up_fragment_id}}->{{down_fragment_id}}","metric":"","query":"rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fragment-level Remote Exchange Recv Throughput","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming Exchange","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":87,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":88,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compute Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":89,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":90,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_reader_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{error_type}}: {{error_msg}} ({{executor_name}}: actor_id={{actor_id}}, source_id={{source_id}})","metric":"","query":"sum(user_source_reader_error{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, actor_id, source_id, executor_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Reader Errors by Type","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":91,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, executor_id, error_msg)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{connector_name}}: {{error_msg}} ({{executor_id}})","metric":"","query":"sum(user_sink_error{job=~\"$job\",instance=~\"$node\"}) by (connector_name, executor_id, error_msg)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink by Connector","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Streaming Errors","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":92,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":93,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Exchange Recv Row Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":94,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_task_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mpp Task Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"All memory usage of batch executors in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":95,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_total_mem{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Mem Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":96,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"batch_heartbeat_worker_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Heartbeat Worker Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":97,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(batch_row_seq_scan_next_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"row_seq_scan next avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(batch_row_seq_scan_next_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Row SeqScan Next Duration","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":98,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":99,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{table_id}} @ {{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Hummock has three parts of memory usage: 1. Meta Cache 2. Block CacheThis metric shows the real memory usage of each of these three caches.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":100,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"data cache - {{job}} @ {{instance}}","metric":"","query":"avg(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":101,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='meta_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block cache miss rate - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"(sum(rate(state_store_sst_store_block_request_counts{type='data_miss',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)) / (sum(rate(state_store_sst_store_block_request_counts{type='data_total',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Cache Miss Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":102,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{table_id}} @ {{type}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_scan_key_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type, table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iter keys flow","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":103,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p50 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts p99 - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts pmax - {{table_id}} @ {{job}} @ {{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_merge_sstable_counts_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, table_id, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"# merged ssts avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_iter_merge_sstable_counts_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Merged SSTs","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the latency of Get operations that have been issued to the state store.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":104,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_get_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of the time spent on iterator initialization.Histogram of the time spent on iterator scanning.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":105,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_init_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"create_iter_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_init_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_init_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_scan_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pure_scan_time avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_iter_scan_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_iter_scan_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Duration - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":106,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive count - {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter check count- {{table_id}} - {{type}}","metric":"","query":"sum(irate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":107,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter positive rate - {{table_id}} - {{type}}","metric":"","query":"(sum(rate(state_store_read_req_bloom_filter_positive_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"False-Positive / Total","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":108,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read req bloom filter false positive rate - {{table_id}} - {{type}}","metric":"","query":"(((sum(rate(state_store_read_req_positive_but_non_exist_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type))) / (sum(rate(state_store_read_req_check_bloom_filter_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id,type)))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Bloom Filter False-Positive Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":109,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_slow_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":110,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer hit - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_shared_buffer_hit_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_in_process_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":111,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":112,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Size - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":113,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.5, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.5, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(0.99, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(0.99, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_iter_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id) + sum((histogram_quantile(1.0, sum(rate(state_store_get_key_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) + histogram_quantile(1.0, sum(rate(state_store_get_value_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Read Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":114,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_item_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Item Count - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of a single key-value pair when reading by operation Get.Operation Get gets a single key-value pair with respect to a caller-specified key. If the key does not exist in the storage, the size of key is counted into this metric and the size of value is 0.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":115,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_get_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance) + sum(rate(state_store_get_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Get","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size of all the key-value paris when reading by operation Iter.Operation Iter scans a range of key-value pairs.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":116,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_iter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Read Throughput - Iter","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":117,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_iter_fetch_meta_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fetch_meta_duration avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id) (rate(state_store_iter_fetch_meta_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":118,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"state_store_iter_fetch_meta_cache_unhits{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Fetch Meta Unhits","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Read)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":11},"height":null,"hideTimeOverride":false,"id":119,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the real memory usage of uploader.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":120,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading memory - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"uploading task size - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_uploader_uploading_task_size{job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader Memory Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Histogram of time spent on compacting shared buffer to remote storage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":121,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Build and Sync Sstable Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":122,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p50 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.5, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write p99 - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(0.99, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write pmax - materialized view {{materialized_view_id}}","metric":"","query":"sum(histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id)) * on(table_id) group_left(materialized_view_id) (group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Write Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":123,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"merge imm tasks - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_merge_imm_task_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploader spill tasks - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(irate(state_store_spill_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Tasks Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":124,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Merging tasks memory size - {{table_id}} @ {{instance}}","metric":"","query":"sum(rate(state_store_merge_imm_memory_sz{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Uploading tasks size - {{uploader_stage}} @ {{instance}}","metric":"","query":"sum(rate(state_store_spill_task_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,uploader_stage)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Uploader - Task Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":125,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write batch - {{table_id}} @ {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"l0 - {{job}} @ {{instance}} ","metric":"","query":"sum(rate(state_store_sync_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":126,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p50 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer p99 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_duration_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to shared_buffer avg - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance, table_id)(rate(state_store_write_batch_duration_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_shared_buffer_sync_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write to object_store - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(state_store_write_shared_buffer_sync_time_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":127,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write_batch_kv_pair_count - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_write_batch_tuple_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Item Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":128,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"shared_buffer - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_write_batch_size_sum{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id) / sum(rate(state_store_write_batch_size_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sync - {{job}} @ {{instance}}","metric":"","query":"sum(rate(compactor_shared_buffer_to_sstable_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance) / sum(rate(compactor_shared_buffer_to_sstable_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric shows the statistics of mem_table size on flush. By default only max (p100) is shown.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":129,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_write_batch_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, table_id, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{table_id}} {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_write_batch_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, table_id, job, instance) (rate(state_store_write_batch_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":130,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mem table spill table id - {{table_id}} @ {{instance}}","metric":"","query":"sum(irate(state_store_mem_table_spill_counts{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Mem Table Spill Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":131,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Sync Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock (Write)","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":12},"height":null,"hideTimeOverride":false,"id":132,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":133,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The size(KB) of SSTables at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":134,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"SSTable Size(KB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The of bytes that have been written by commit epoch per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":135,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{table_id}}","metric":"","query":"sum(rate(storage_commit_write_throughput{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Flush Bytes by Table","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":136,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result!='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Failure Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have completed or failed","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":137,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task_type}} - {{result}} - group-{{group}} @ {{compactor}}","metric":"","query":"sum(storage_level_compact_frequency{result='SUCCESS',job=~\"$job\",instance=~\"$node\"}) by (compactor, group, task_type, result)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Success Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that have been skipped.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":138,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{level}}-{{type}}","metric":"","query":"sum(rate(storage_skip_compact_frequency{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (level, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Skip Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg l0 select_level_count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":139,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_l0_compact_level_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_l0_compact_level_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task L0 Select Level Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg file count of the compact task, and categorize it according to different cg, levels and task types","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":140,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg cg{{group}}@{{type}}","metric":"","query":"sum by(le, group, type)(irate(storage_compact_task_file_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, group, type)(irate(storage_compact_task_file_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task File Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The distribution of the compact task size triggered, including p90 and max. and categorize it according to different cg, levels and task types.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":141,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - cg{{group}}@{{type}}","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_compact_task_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Task Size Distribution","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of compactions from one level to another level that are running.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":142,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_count - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_num{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor_task_pending_parallelism - {{job}} @ {{instance}}","metric":"","query":"avg(storage_compact_task_pending_parallelism{job=~\"$job\",instance=~\"$node\"}) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Running Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compact-task: The total time have been spent on compaction.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":143,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_task_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compact_sst_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get-table-id pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_get_table_id_total_time_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{job}}","metric":"","query":"histogram_quantile(0.9, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{job}}","metric":"","query":"histogram_quantile(1.0, sum(rate(compactor_remote_read_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute_apply_version_duration_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(compute_refill_cache_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-task avg","metric":"","query":"sum by(le)(rate(compactor_compact_task_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(compactor_compact_task_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact-key-range avg","metric":"","query":"sum by(le)(rate(state_store_compact_sst_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le)(rate(state_store_compact_sst_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"KBs read from next level during history compactions to next level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":144,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job) + sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(rate(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fast compact - {{job}}","metric":"","query":"sum(rate(compactor_fast_compact_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Write refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":145,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Bytes(GiB)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Write amplification is the amount of bytes written to the remote storage by compaction for each one byte of flushed SSTable data. Write amplification is by definition higher than 1.0 because we write each piece of data to L0, and then write it again to an SSTable, and then compaction may read this piece of data and write it to a new SSTable, that's another write.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":146,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write amplification","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) / sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"})","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Write Amplification","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of SSTables that is being compacted at each level","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":147,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"L{{level_index}}","metric":"","query":"storage_level_compact_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting SSTable Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"num of compact_task","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":148,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{task}}","metric":"","query":"storage_level_compact_task_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compacting Task Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":149,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(rate(storage_level_compact_read_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(rate(storage_level_compact_read_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(rate(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"KBs Read/Write by Level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":150,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} write to next level","metric":"","query":"sum(irate(storage_level_compact_write_sstn{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from next level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_next{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"cg{{group}}-L{{level_index}} read from current level","metric":"","query":"sum(irate(storage_level_compact_read_sstn_curr{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, group, level_index)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Count of SSTs Read/Write by level","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_bloom_filter, for observing bloom_filter size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":151,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_meta - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_bloom_filter_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_file - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_file_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_file_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total bytes gotten from sstable_avg_key_size, for observing sstable_avg_key_size","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":72},"height":null,"hideTimeOverride":false,"id":152,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_key_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_key_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_value_size - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_avg_value_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Item Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Avg count gotten from sstable_distinct_epoch_count, for observing sstable_distinct_epoch_count","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":80},"height":null,"hideTimeOverride":false,"id":153,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg_epoch_count - {{job}} @ {{instance}}","metric":"","query":"sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job, instance)(rate(compactor_sstable_distinct_epoch_count_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Sstable Stat","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total time of operations which read from remote storage when enable prefetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":80},"height":null,"hideTimeOverride":false,"id":154,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io p90 - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote-io pmax - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(state_store_remote_read_time_per_task_bucket{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance, table_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hummock Remote Read Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":88},"height":null,"hideTimeOverride":false,"id":155,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"iter keys flow - {{type}} @ {{instance}} ","metric":"","query":"sum(rate(compactor_iter_scan_key_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compactor Iter keys","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"bytes of Lsm tree needed to reach balance","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":88},"height":null,"hideTimeOverride":false,"id":156,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compact pending bytes - {{group}} @ {{instance}} ","metric":"","query":"sum(storage_compact_pending_bytes{job=~\"$job\",instance=~\"$node\"}) by (instance, group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Compact Pending Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"compression ratio of each level of the lsm tree","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":96},"height":null,"hideTimeOverride":false,"id":157,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"lsm compression ratio - cg{{group}} @ L{{level}} - {{algorithm}} {{instance}}","metric":"","query":"sum(storage_compact_level_compression_ratio{job=~\"$job\",instance=~\"$node\"}) by (instance, group, level, algorithm)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lsm Level Compression Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Compaction","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":13},"height":null,"hideTimeOverride":false,"id":158,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":159,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":160,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_latency_bucket{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} avg - {{job}} @ {{instance}}","metric":"","query":"sum by(le, type, job, instance)(rate(object_store_operation_latency_sum{type!~'streaming_upload_write_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(object_store_operation_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":161,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type!~'streaming_upload_write_bytes|streaming_read_read_bytes|streaming_read',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-write - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'upload|delete',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{media_type}}-read - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_operation_latency_count{type=~'read|readv|list|metadata',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, media_type, job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":162,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(object_store_operation_bytes_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, type, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":163,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Failure Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":164,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(aws_sdk_retry_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} - {{job}} @ {{instance}}","metric":"","query":"sum(irate(s3_read_request_retry_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Operation Retry Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"There are two types of operations: 1. GET, SELECT, and DELETE, they cost 0.0004 USD per 1000 requests. 2. PUT, COPY, POST, LIST, they cost 0.005 USD per 1000 requests.Reading from S3 across different regions impose extra cost. This metric assumes 0.01 USD per 1GB data transfer. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":165,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","format":"time_series","hide":true,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"(Cross Region) Data Transfer Cost","metric":"","query":"sum(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}) * 0.01 / 1000 / 1000 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GET, SELECT, and all other Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'read|streaming_read_start|delete',job=~\"$job\",instance=~\"$node\"}) * 0.0004 / 1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"PUT, COPY, POST, LIST Requests Cost","metric":"","query":"sum(object_store_operation_latency_count{type=~'upload|streaming_upload_start|s3_upload_part|streaming_upload_finish|delete_objects|list',job=~\"$job\",instance=~\"$node\"}) * 0.005 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Realtime)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"This metric uses the total size of data in S3 at this second to derive the cost of storing data for a whole month. The price is 0.023 USD per GB. Please checkout AWS's pricing model for more accurate calculation.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"$"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":166,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Monthly Storage Cost","metric":"","query":"sum(storage_level_total_file_size{job=~\"$job\",instance=~\"$node\"}) by (instance) * 0.023 / 1000 / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Estimated S3 Cost (Monthly)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Object Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":14},"height":null,"hideTimeOverride":false,"id":167,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":168,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":169,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":170,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":171,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache hit ratio @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) / (sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance) + sum(rate(foyer_storage_op_duration_count{op=\"lookup\",extra=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Hit Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":172,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache refill - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=~\"meta|data\",op!~\"filtered|ignored\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (type, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":173,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{type}} file cache - {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Data Refill Throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":174,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} cache refill - {{op}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(refill_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":175,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(refill_queue_total) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"refill queue length @ {{instance}}","metric":"","query":"sum(refill_queue_total) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Refill Queue Length","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":176,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} size @ {{instance}}","metric":"","query":"sum(foyer_storage_total_bytes{job=~\"$job\",instance=~\"$node\"}) by (foyer, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":177,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_inner_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inner Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":178,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{foyer}} file cache {{op}} {{extra}} @ {{instance}}","metric":"","query":"sum(rate(foyer_storage_slow_op_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (foyer, op, extra, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":179,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax - {{foyer}} file cache - {{op}} {{extra}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(rate(foyer_storage_slow_op_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, foyer, op, extra, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Slow Op Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":180,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":181,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parent meta lookup hit ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"parent_meta\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"parent_meta\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Parent Meta Lookup Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":182,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":183,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unit inheritance ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / (sum(rate(refill_total{type=\"unit_inheritance\",op=\"hit\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) + sum(rate(refill_total{type=\"unit_inheritance\",op=\"miss\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inheritance - Unit inheritance Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":184,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill {{op}} @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":185,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"block refill ratio @ {{instance}}","metric":"","query":"sum(rate(refill_total{type=\"block\",op=\"success\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance) / sum(rate(refill_total{type=\"block\",op=\"unfiltered\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Block Refill Ratio","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Tiered Cache","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":15},"height":null,"hideTimeOverride":false,"id":186,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":187,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p50 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.5, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time p99 - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(0.99, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lock Time pmax - {{lock_type}} @ {{lock_name}}","metric":"","query":"histogram_quantile(1.0, sum(rate(hummock_manager_lock_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, lock_name, lock_type))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Lock Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":188,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p50 - {{method}}","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time p99 - {{method}}","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Real Process Time pmax - {{method}}","metric":"","query":"histogram_quantile(1.0, sum(rate(meta_hummock_manager_real_process_time_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, method))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Real Process Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":189,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version size","metric":"","query":"storage_version_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":190,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"current version id","metric":"","query":"storage_current_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"checkpoint version id","metric":"","query":"storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned version id","metric":"","query":"storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min safepoint version id","metric":"","query":"storage_min_safepoint_version_id{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Id","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":191,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"max committed epoch","metric":"","query":"storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"safe epoch","metric":"","query":"storage_safe_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"min pinned epoch","metric":"","query":"storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":192,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_value_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":193,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":194,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table{{table_id}} {{metric}}","metric":"","query":"storage_version_stats{metric='total_key_count',table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Table KV Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\nObjects are classified into 3 groups:\n- not referenced by versions: these object are being deleted from object store.\n- referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n- referenced by current version: these objects are in the latest version.\n\nAdditionally, a metric on all objects (including dangling ones) is updated with low-frequency. The metric is updated right before full GC. So subsequent full GC may reduce the actual value significantly, without updating the metric.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":195,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects (including dangling ones)","metric":"","query":"storage_total_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Refer to `Object Total Number` panel for classification of objects.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":196,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"all objects, including dangling ones","metric":"","query":"storage_total_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of hummock version delta log","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":197,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"delta log total number","metric":"","query":"storage_delta_log_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Delta Log Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"hummock version checkpoint latency","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":198,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p90","metric":"","query":"histogram_quantile(0.9, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_p999","metric":"","query":"histogram_quantile(0.999, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_pmax","metric":"","query":"histogram_quantile(1.0, sum(rate(storage_version_checkpoint_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"version_checkpoint_latency_avg","metric":"","query":"rate(storage_version_checkpoint_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(storage_version_checkpoint_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Version Checkpoint Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"When certain per compaction group threshold is exceeded (e.g. number of level 0 sub-level in LSMtree), write op to that compaction group is stopped temporarily. Check log for detail reason of write stop.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":199,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compaction_group_{{compaction_group_id}}","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Stop Compaction Groups","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"total number of attempts to trigger full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":200,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_trigger_count","metric":"","query":"storage_full_gc_trigger_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Trigger Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"the object id watermark used in last full GC","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":201,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"full_gc_last_object_id_watermark","metric":"","query":"storage_full_gc_last_object_id_watermark{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Full GC Last Watermark","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":202,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"meta iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(storage_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor consumed latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_consumed_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compactor iteration latency pmax - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(1.0, sum(irate(compactor_compaction_event_loop_iteration_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Compaction Event Loop Time","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The times of move_state_table occurs","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":203,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"move table cg{{group}}","metric":"","query":"sum(storage_move_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Move State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of state_tables in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":204,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"state table cg{{group}}","metric":"","query":"sum(irate(storage_state_table_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of branched_sst in each CG","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":72},"height":null,"hideTimeOverride":false,"id":205,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"branched sst cg{{group}}","metric":"","query":"sum(irate(storage_branched_sst_count{table_id=~\"$table|\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (group)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Branched SST Count","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":206,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total backup job count since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":207,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"job count","metric":"","query":"backup_job_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Latency of backup jobs since the Meta node starts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":208,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p50 - {{state}}","metric":"","query":"histogram_quantile(0.5, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time p99 - {{state}}","metric":"","query":"histogram_quantile(0.99, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Job Process Time pmax - {{state}}","metric":"","query":"histogram_quantile(1.0, sum(rate(backup_job_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, state))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Job Process Time","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Backup Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":17},"height":null,"hideTimeOverride":false,"id":209,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":210,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Create_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Create',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":211,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Drop_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/Drop',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Drop latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":212,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetCatalog_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.CatalogService/GetCatalog',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetCatalog latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Catalog Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":213,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":214,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"AddWorkerNode_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/AddWorkerNode',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"AddWorkerNode latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":215,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ListAllNodes_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.ClusterService/ListAllNodes',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ListAllNodes latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Cluster Service","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":19},"height":null,"hideTimeOverride":false,"id":216,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":217,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"CreateMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/CreateMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"CreateMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":218,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"DropMaterializedView_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/DropMaterializedView',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"DropMaterializedView latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":219,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.StreamManagerService/Flush',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Flush latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Stream Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":20},"height":null,"hideTimeOverride":false,"id":220,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":221,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinVersionBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinVersionBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinVersionBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":8,"y":0},"height":null,"hideTimeOverride":false,"id":222,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"UnpinSnapshotBefore_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/UnpinSnapshotBefore',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UnpinSnapshotBefore latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":16,"y":0},"height":null,"hideTimeOverride":false,"id":223,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"ReportCompactionTasks_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/ReportCompactionTasks',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"ReportCompactionTasks latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":8,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":224,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p50","metric":"","query":"histogram_quantile(0.5, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p90","metric":"","query":"histogram_quantile(0.9, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_p99","metric":"","query":"histogram_quantile(0.99, sum(irate(meta_grpc_duration_seconds_bucket{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"GetNewSstIds_avg","metric":"","query":"sum(irate(meta_grpc_duration_seconds_sum{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(meta_grpc_duration_seconds_count{path='/meta.HummockManagerService/GetNewSstIds',job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"GetNewSstIds latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC Meta: Hummock Manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":21},"height":null,"hideTimeOverride":false,"id":225,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":226,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_counts - {{instance}}","metric":"","query":"sum(irate(state_store_report_compaction_task_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":227,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_avg","metric":"","query":"sum(irate(state_store_unpin_version_before_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_version_before_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_version_before_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_version_before_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"version_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":228,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latencyp90 - {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(irate(state_store_pin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_pin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_pin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_unpin_version_snapshot_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_avg","metric":"","query":"sum(irate(state_store_unpin_snapshot_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_unpin_snapshot_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_unpin_snapshot_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":229,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_pin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"unpin_snapshot_counts - {{instance}}","metric":"","query":"sum(irate(state_store_unpin_snapshot_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"snapshot_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":230,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_avg","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_get_new_sst_ids_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_get_new_sst_ids_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":231,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"get_new_sst_ids_latency_counts - {{instance}}","metric":"","query":"sum(irate(state_store_get_new_sst_ids_latency_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))by(job, instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"table_count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":232,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_avg","metric":"","query":"sum(irate(state_store_report_compaction_task_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum(irate(state_store_report_compaction_task_latency_count[$__rate_interval]))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"report_compaction_task_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(state_store_report_compaction_task_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"compaction_latency","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"gRPC: Hummock Meta Client","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":22},"height":null,"hideTimeOverride":false,"id":233,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":234,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":235,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":236,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Per Second (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":237,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Running Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":238,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Rejected queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":239,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The Number of Completed Queries (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":240,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Distributed Query Mode)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":241,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency (Local Query Mode)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Frontend","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":23},"height":null,"hideTimeOverride":false,"id":242,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":243,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"rate(lru_runtime_loop_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager loop count per sec","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":244,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_watermark_step{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager watermark steps","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"watermark_time is the current lower watermark of cached data. physical_now is the current time of the machine. The diff (physical_now - watermark_time) shows how much data is cached.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":245,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"lru_physical_now_ms{job=~\"$job\",instance=~\"$node\"} - lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between watermark_time and now (ms)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":246,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":247,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":248,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_resident_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The resident memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":249,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jemalloc_metadata_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The metadata memory of jemalloc","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":250,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_allocated_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The allocated memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":251,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"jvm_active_bytes{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"The active memory of jvm","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":252,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"table {{table_id}} actor {{actor_id}} desc: {{desc}}","metric":"","query":"lru_current_watermark_time_ms{job=~\"$job\",instance=~\"$node\"} - on() group_right() lru_evicted_watermark_time_ms{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"LRU manager diff between current watermark and evicted watermark time (ms) for actors","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory manager","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":253,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":254,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":255,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":25},"height":null,"hideTimeOverride":false,"id":256,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":257,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{connector}} {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(sink_commit_duration_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, connector, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{connector}} @ {{sink_id}}","metric":"","query":"sum by(le, connector, sink_id)(rate(sink_commit_duration_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(sink_commit_duration_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Commit Duration","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":258,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest write epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"latest read epoch @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Read/Write Epoch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":259,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"(max(log_store_latest_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":260,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Consume persistent log lag @ {{connector}} {{sink_id}} {{executor_id}}","metric":"","query":"clamp_min((max(log_store_first_write_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)- max(log_store_latest_read_epoch{job=~\"$job\",instance=~\"$node\"}) by (connector, sink_id, executor_id)) / (2^16) / 1000, 0)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Persistent Log Lag","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":261,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":262,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_read_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Consume Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":263,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":264,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector}} {{sink_id}} @ {{executor_id}} {{instance}}","metric":"","query":"sum(rate(log_store_write_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, connector, sink_id, executor_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Log Store Write Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":265,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":266,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_read_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Read Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":267,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Row Ops","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":268,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_storage_write_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Write Storage Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":269,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"sum(rate(kv_log_store_rewind_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (executor_id, connector, sink_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Kv Log Store Rewind Rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":270,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} - {{connector}} @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(kv_log_store_rewind_delay_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, executor_id, connector, sink_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rewind delay (second)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":271,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current number of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":272,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Current total size of messages in producer queues","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":273,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_msg_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Size in Producer Queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages transmitted (produced) to Kafka brokers","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":274,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_tx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Produced Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of messages consumed, not including ignored messages (due to offset, etc), from Kafka brokers.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":275,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id }}","metric":"","query":"rdkafka_top_rx_msgs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Received Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages awaiting transmission to broker","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":276,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_outbuf_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message Count Pending to Transmit (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages in-flight to broker awaiting response","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":277,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_waitresp_msg_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Inflight Message Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of transmission errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":278,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_tx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Transmitting (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of receive errors","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":279,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_rx_errs{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Error Count When Receiving (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of requests timed out","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":32},"height":null,"hideTimeOverride":false,"id":280,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, state {{ state }}","metric":"","query":"rdkafka_broker_req_timeouts{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Timeout Request Count (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker latency / round-trip time in milli seconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":32},"height":null,"hideTimeOverride":false,"id":281,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_rtt_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"RTT (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Broker throttling time in milliseconds","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":40},"height":null,"hideTimeOverride":false,"id":282,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_avg{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p75{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p90{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_p99_99{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}","metric":"","query":"rdkafka_broker_throttle_out_of_range{job=~\"$job\",instance=~\"$node\"}/1000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Throttle Time (per broker)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Age of metadata from broker for this topic (milliseconds)","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ms"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":40},"height":null,"hideTimeOverride":false,"id":283,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}","metric":"","query":"rdkafka_topic_metadata_age{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Metadata_age Age","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch sizes in bytes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":48},"height":null,"hideTimeOverride":false,"id":284,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchsize_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Batch message counts","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":null,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_avg{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p75{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p90{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_p99_99{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, broker {{ broker }}, topic {{ topic }}","metric":"","query":"rdkafka_topic_batchcnt_out_of_range{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Messages","transformations":[],"transparent":false,"type":"timeseries"}],"timeFrom":null,"timeShift":null,"title":"Topic Batch Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of messages ready to be produced in transmit queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":285,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_xmit_msgq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message to be Transmitted","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of pre-fetched messages in fetch queue","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":56},"height":null,"hideTimeOverride":false,"id":286,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_fetchq_cnt{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Message in pre fetch queue","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Next offset to fetch","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":64},"height":null,"hideTimeOverride":false,"id":287,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_next_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Next offset to fetch","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Last committed offset","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":64},"height":null,"hideTimeOverride":false,"id":288,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"id {{ id }}, client_id {{ client_id}}, topic {{ topic }}, partition {{ partition }}","metric":"","query":"rdkafka_topic_partition_committed_offset{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Committed Offset","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Kafka Native Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":27},"height":null,"hideTimeOverride":false,"id":289,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":290,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Network throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":291,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"S3 throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":292,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total read @ {{instance}}","metric":"","query":"sum(rate(connection_read_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total write @ {{instance}}","metric":"","query":"sum(rate(connection_write_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"gRPC throughput","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":293,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(irate(connection_io_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} grpc {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} total {{op_type}} err[{{error_kind}}] @ {{instance}}","metric":"","query":"sum(rate(connection_io_err_rate{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, op_type, error_kind)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"IO error rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":294,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(connection_count{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Existing connection count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":295,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_create_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection rate","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":296,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} S3 @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=\"S3\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} {{connection_type}} @ {{instance}}","metric":"","query":"sum(irate(connection_err_rate{connection_type=~\"grpc.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, connection_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Create new connection err rate","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network connection","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":28},"height":null,"hideTimeOverride":false,"id":297,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"iceberg write qps","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":298,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_write_qps{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Qps Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":299,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.5, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 @ {{sink_id}}","metric":"","query":"histogram_quantile(0.99, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"pmax @ {{sink_id}}","metric":"","query":"histogram_quantile(1.0, sum(rate(iceberg_write_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, sink_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg @ {{sink_id}}","metric":"","query":"sum by(le, sink_id)(rate(iceberg_write_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, type, job, instance) (rate(iceberg_write_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Latency Of Iceberg Writer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":300,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_rolling_unfushed_data_file{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg rolling unfushed data file","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":301,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_position_delete_cache_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg position delete cache num","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":302,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{executor_id}} @ {{sink_id}}","metric":"","query":"iceberg_partition_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Iceberg partition num","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Iceberg Sink Metrics","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":29},"height":null,"hideTimeOverride":false,"id":303,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":304,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{instance}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{instance}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_success_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_success_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_failure_count - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Calls Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":305,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_input_chunk_rows_avg - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_input_chunk_rows_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_input_chunk_rows_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Input Chunk Rows","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":306,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p50 - {{instance}}","metric":"","query":"histogram_quantile(0.50, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p90 - {{instance}}","metric":"","query":"histogram_quantile(0.90, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99 - {{instance}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg - {{instance}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_p99_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"histogram_quantile(0.99, sum(irate(udf_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, link, name, fragment_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_latency_avg_by_name - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(irate(udf_latency_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / sum(irate(udf_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":307,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{instance}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_rows - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_rows{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":308,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{instance}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance) / (1024*1024)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"udf_throughput_bytes - {{link}} {{name}} {{fragment_id}}","metric":"","query":"sum(rate(udf_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (link, name, fragment_id) / (1024*1024)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"UDF Throughput (bytes)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"User Defined Function","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(table_info, table_id)","description":"Reporting table id of the metric","hide":0,"includeAll":true,"label":"Table","multi":true,"name":"table","options":[],"query":{"query":"label_values(table_info, table_id)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dev_dashboard","uid":"Ecy3uV1nz","version":0} diff --git a/grafana/risingwave-user-dashboard.dashboard.py b/grafana/risingwave-user-dashboard.dashboard.py index f547ab04b9b5f..5b48a1e283fbd 100644 --- a/grafana/risingwave-user-dashboard.dashboard.py +++ b/grafana/risingwave-user-dashboard.dashboard.py @@ -31,26 +31,26 @@ logging.basicConfig(level=logging.WARN) -def section_actor_info(panels): - excluded_cols = ["Time", "Value", "__name__", f"{COMPONENT_LABEL}", f"{NODE_LABEL}"] +def section_actor_info(outer_panels): + panels = outer_panels.sub_panel() return [ - panels.row("Actor/Table Id Info"), - panels.table_info( - "Actor Id Info", - "Mapping from actor id to fragment id", - [panels.table_target(f"{metric('actor_info')}")], - excluded_cols, - ), - panels.table_info( - "Materialized View Info", - "Mapping from materialized view table id to it's internal table ids", + outer_panels.row_collapsed( + "Actor/Table Id Info", [ - panels.table_target( - f"group({metric('table_info')}) by (materialized_view_id, table_id, table_name, table_type)" - ) + panels.table_info( + "Actor Info", + "Information about actors", + [panels.table_target(f"group({metric('actor_info')}) by (actor_id, fragment_id, compute_node)")], + ["actor_id", "fragment_id", "compute_node"], + ), + panels.table_info( + "State Table Info", + "Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.", + [panels.table_target(f"group({metric('table_info')}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)")], + ["table_id", "table_name", "table_type", "materialized_view_id", "fragment_id", "compaction_group_id"], + ), ], - excluded_cols, - ), + ) ] @@ -60,22 +60,22 @@ def section_overview(panels): return [ panels.row("Overview"), panels.timeseries_rowsps( - "Aggregated Source Throughput(rows/s)", + "Source Throughput(rows/s)", "The figure shows the number of rows read by each source per second.", [ panels.target( - f"sum(rate({metric('stream_source_output_rows_counts')}[$__rate_interval])) by (source_name)", - "{{source_name}}", + f"sum(rate({metric('stream_source_output_rows_counts')}[$__rate_interval])) by (source_id, source_name, fragment_id)", + "{{source_id}} {{source_name}} (fragment {{fragment_id}})", ), ], ), panels.timeseries_bytesps( - "Aggregated Source Throughput(MB/s)", + "Source Throughput(MB/s)", "The figure shows the number of bytes read by each source per second.", [ panels.target( - f"(sum by (source_id)(rate({metric('partition_input_bytes')}[$__rate_interval])))/(1000*1000)", - "source_id {{source_id}}", + f"(sum by (source_id, source_name, fragment_id)(rate({metric('source_partition_input_bytes')}[$__rate_interval])))/(1000*1000)", + "{{source_id}} {{source_name}} (fragment {{fragment_id}})", ) ], ), diff --git a/grafana/risingwave-user-dashboard.json b/grafana/risingwave-user-dashboard.json index d180e0e4b16c5..f9ec8cab9831f 100644 --- a/grafana/risingwave-user-dashboard.json +++ b/grafana/risingwave-user-dashboard.json @@ -1 +1 @@ -{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from actor id to fragment id","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"actor_info{job=~\"$job\",instance=~\"$node\"}","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Id Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Mapping from materialized view table id to it's internal table ids","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]}},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":1},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true},"repeat":null,"repeatDirection":null,"span":6,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (materialized_view_id, table_id, table_name, table_type)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true,"__name__":true,"instance":true,"job":true}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":9},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source_id {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Aggregated Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n - Write Stall: Compaction cannot keep up. Stall foreground write.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":42},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":50},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":51},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":52},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":53},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":54},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":55},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":56},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"mappings":[],"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} +{"__inputs":[],"annotations":{"list":[]},"description":"RisingWave Dashboard","editable":true,"gnetId":null,"graphTooltip":0,"hideControls":false,"id":null,"links":[],"panels":[{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":1,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about actors","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":2,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(actor_info{job=~\"$job\",instance=~\"$node\"}) by (actor_id, fragment_id, compute_node)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"actor_id":0,"compute_node":2,"fragment_id":1}}}],"transparent":false,"type":"table"},{"cacheTimeout":null,"color":{"mode":"thresholds"},"columns":[],"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Information about state tables. Column `materialized_view_id` is the id of the materialized view that this state table belongs to.","editable":true,"error":false,"fieldConfig":{"defaults":{"custom":{"align":"auto","displayMode":"auto","filterable":true},"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"fontSize":"100%","gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":3,"interval":null,"links":[],"mappings":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"options":{"showHeader":true,"sortBy":[]},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","format":"table","hide":false,"instant":true,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name, table_type, materialized_view_id, fragment_id, compaction_group_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"State Table Info","transformations":[{"id":"organize","options":{"excludeByName":{"Time":true,"Value":true},"indexByName":{"compaction_group_id":5,"fragment_id":4,"materialized_view_id":3,"table_id":0,"table_name":1,"table_type":2}}}],"transparent":false,"type":"table"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Actor/Table Id Info","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":false,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":1},"height":null,"hideTimeOverride":false,"id":4,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Overview","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":2},"height":null,"hideTimeOverride":false,"id":5,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"sum(rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (source_id, source_name, fragment_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":2},"height":null,"hideTimeOverride":false,"id":6,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{source_id}} {{source_name}} (fragment {{fragment_id}})","metric":"","query":"(sum by (source_id, source_name, fragment_id)(rate(source_partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of rows streamed into each sink per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":10},"height":null,"hideTimeOverride":false,"id":7,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink {{sink_id}} {{sink_name}}","metric":"","query":"sum(rate(stream_sink_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (sink_id) * on(sink_id) group_left(sink_name) group(sink_info{job=~\"$job\",instance=~\"$node\"}) by (sink_id, sink_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Sink Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized view per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":10},"height":null,"hideTimeOverride":false,"id":8,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"mview {{table_id}} {{table_name}}","metric":"","query":"sum(rate(stream_mview_input_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id) * on(table_id) group_left(table_name) group(table_info{job=~\"$job\",instance=~\"$node\"}) by (table_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The time that the data between two consecutive barriers gets fully processed, i.e. the computation results are made durable into materialized views or sink to external systems. This metric shows to users the freshness of materialized views.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":18},"height":null,"hideTimeOverride":false,"id":9,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p50","metric":"","query":"histogram_quantile(0.5, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_p99","metric":"","query":"histogram_quantile(0.99, sum(rate(meta_barrier_duration_seconds_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"barrier_latency_avg","metric":"","query":"rate(meta_barrier_duration_seconds_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) / rate(meta_barrier_duration_seconds_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Barrier Latency","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Alerts in the system group by type:\n - Too Many Barriers: there are too many uncommitted barriers generated. This means the streaming graph is stuck or under heavy load. Check 'Barrier Latency' panel.\n - Recovery Triggered: cluster recovery is triggered. Check 'Errors by Type' / 'Node Count' panels.\n - Lagging Version: the checkpointed or pinned version id is lagging behind the current version id. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Epoch: the pinned or safe epoch is lagging behind the current max committed epoch. Check 'Hummock Manager' section in dev dashboard.\n - Lagging Compaction: there are too many files in L0. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Lagging Vacuum: there are too many stale files waiting to be cleaned. This can be caused by compactor failure or lag of compactor resource. Check 'Compaction' section in dev dashboard.\n - Abnormal Meta Cache Memory: the meta cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Block Cache Memory: the block cache memory usage is too large, exceeding the expected 10 percent.\n - Abnormal Uploading Memory Usage: uploading memory is more than 70 percent of the expected, and is about to spill.\n - Write Stall: Compaction cannot keep up. Stall foreground write.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":18},"height":null,"hideTimeOverride":false,"id":10,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Too Many Barriers","metric":"","query":"all_barrier_nums{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recovery Triggered","metric":"","query":"sum(rate(recovery_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) > bool 0 + sum(recovery_failure_cnt{job=~\"$job\",instance=~\"$node\"}) > bool 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Version","metric":"","query":"((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_checkpoint_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100) + ((storage_current_version_id{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_version_id{job=~\"$job\",instance=~\"$node\"}) >= bool 100)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Epoch","metric":"","query":"((storage_max_committed_epoch{job=~\"$job\",instance=~\"$node\"} - storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"}) >= bool 6553600000 unless + storage_min_pinned_epoch{job=~\"$job\",instance=~\"$node\"} == 0)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Compaction","metric":"","query":"sum(label_replace(storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}, 'L0', 'L0', 'level_index', '.*_L0') unless storage_level_sst_num{job=~\"$job\",instance=~\"$node\"}) by (L0) >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lagging Vacuum","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"} >= bool 200","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Meta Cache Memory","metric":"","query":"state_store_meta_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Block Cache Memory","metric":"","query":"state_store_block_cache_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 1.1","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Abnormal Uploading Memory Usage","metric":"","query":"state_store_uploading_memory_usage_ratio{job=~\"$job\",instance=~\"$node\"} >= bool 0.7","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Write Stall","metric":"","query":"storage_write_stop_compaction_groups{job=~\"$job\",instance=~\"$node\"} > bool 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Alerts","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Errors in the system group by type","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":26},"height":null,"hideTimeOverride":false,"id":11,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"compute error {{error_type}}: {{error_msg}} ({{executor_name}}: fragment_id={{fragment_id}})","metric":"","query":"sum(user_compute_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"parse error {{error_type}}: {{error_msg}} ({{executor_name}}: table_id={{table_id}}, fragment_id={{fragment_id}})","metric":"","query":"sum(user_source_error_count{job=~\"$job\",instance=~\"$node\"}) by (error_type, error_msg, fragment_id, table_id, executor_name)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source error: source_id={{source_id}}, source_name={{source_name}} @ {{instance}}","metric":"","query":"source_status_is_up{job=~\"$job\",instance=~\"$node\"} == 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"remote storage error {{type}}: {{job}} @ {{instance}}","metric":"","query":"sum(rate(object_store_failure_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance, job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Errors","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Qps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":26},"height":null,"hideTimeOverride":false,"id":12,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Local mode","metric":"","query":"rate(frontend_query_counter_local_execution{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distributed mode","metric":"","query":"rate(distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Query QPS","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of each type of RisingWave components alive.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":34},"height":null,"hideTimeOverride":false,"id":13,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{worker_type}}","metric":"","query":"sum(worker_num{job=~\"$job\",instance=~\"$node\"}) by (worker_type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Count","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of active sessions in frontend nodes","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":34},"height":null,"hideTimeOverride":false,"id":14,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"","metric":"","query":"frontend_active_sessions{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Active Sessions","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":42},"height":null,"hideTimeOverride":false,"id":15,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The CPU usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":16,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"sum(rate(process_cpu_seconds_total{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Usage","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Number of CPU cores per RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":17,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{instance}}","metric":"","query":"avg(process_cpu_core_num{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node CPU Core Number","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"CPU","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":43},"height":null,"hideTimeOverride":false,"id":18,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The memory usage of each RisingWave component.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":19,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{job}} @ {{instance}}","metric":"","query":"avg(process_resident_memory_bytes{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Node Memory","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":20,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":[],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (instance) + sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Total)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":21,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(actor_memory_usage[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"streaming actor - {{actor_id}}","metric":"","query":"rate(actor_memory_usage[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage meta cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_meta_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage block cache - {{job}} @ {{instance}}","metric":"","query":"sum(state_store_block_cache_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"storage write buffer - {{job}} @ {{instance}}","metric":"","query":"sum(uploading_memory_size{job=~\"$job\",instance=~\"$node\"}) by (job,instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized_view {{materialized_view_id}}","metric":"","query":"sum(stream_memory_usage{job=~\"$job\",instance=~\"$node\"} * on(table_id) group_left(materialized_view_id) table_info) by (materialized_view_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Memory Usage (Detailed)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Executor cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":22,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - cache miss - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Join - total lookups - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Group top n appendonly - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - cache miss - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Lookup executor - total lookups - table {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - cache miss - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Temporal join - total lookups - table_id {{table_id}} actor {{actor_id}}","metric":"","query":"rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - cache hit count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Materialize - total cache count - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":23,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"join executor cache miss ratio - - {{side}} side, join_table_id {{join_table_id}} degree_table_id {{degree_table_id}} actor {{actor_id}}","metric":"","query":"(sum(rate(stream_join_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id) ) / (sum(rate(stream_join_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (side, join_table_id, degree_table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_lookup_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_lookup_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Distinct agg cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_agg_distinct_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_agg_distinct_total_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream group top n appendonly cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_group_top_n_appendonly_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_group_top_n_appendonly_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream lookup cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_lookup_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_lookup_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Stream temporal join cache miss ratio - table {{table_id}} actor {{actor_id}} ","metric":"","query":"(sum(rate(stream_temporal_join_cache_miss_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_temporal_join_total_query_cache_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialize executor cache miss ratio - table {{table_id}} - actor {{actor_id}} {{instance}}","metric":"","query":"1 - (sum(rate(stream_materialize_cache_hit_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id) ) / (sum(rate(stream_materialize_cache_total_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (table_id, actor_id))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Executor Cache Miss Ratio","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":24,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"memory cache - {{table_id}} @ {{type}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, instance, table_id, type)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"total_meta_miss_count - {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_sst_store_block_request_counts{type='meta_miss',job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job, type)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Cache","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage bloom filter statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":24},"height":null,"hideTimeOverride":false,"id":25,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter total - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_check_bloom_filter_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"bloom filter false positive - {{table_id}} @ {{job}} @ {{instance}}","metric":"","query":"sum(rate(state_store_read_req_positive_but_non_exist_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job,instance,table_id)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Bloom Filer","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Storage file cache statistics","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"ops"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":24},"height":null,"hideTimeOverride":false,"id":26,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache {{op}} @ {{instance}}","metric":"","query":"sum(rate(file_cache_latency_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (op, instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"file cache miss @ {{instance}}","metric":"","query":"sum(rate(file_cache_miss{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage File Cache","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Memory","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":44},"height":null,"hideTimeOverride":false,"id":27,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Send/Recv throughput per node for streaming exchange","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":28,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Send @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_send_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Recv @ {{instance}}","metric":"","query":"sum(rate(stream_exchange_frag_recv_size{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Streming Remote Exchange (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput per node","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":29,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{instance}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{instance}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (instance)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"row"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":30,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{query_id}} : {{source_stage_id}}.{{source_task_id}} -> {{target_stage_id}}.{{target_task_id}}","metric":"","query":"batch_exchange_recv_row_number{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Batch Exchange Recv (Rows/s)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Network","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":45},"height":null,"hideTimeOverride":false,"id":31,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":32,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_size{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The storage size of each materialized view","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"kbytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":33,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"{{metric}}, mv id - {{table_id}} ","metric":"","query":"storage_materialized_view_stats{metric='materialized_view_total_size',job=~\"$job\",instance=~\"$node\"}/1024","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Size","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"\n Objects are classified into 3 groups:\n - not referenced by versions: these object are being deleted from object store.\n - referenced by non-current versions: these objects are stale (not in the latest version), but those old versions may still be in use (e.g. long-running pinning). Thus those objects cannot be deleted at the moment.\n - referenced by current version: these objects are in the latest version.\n ","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":34,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"not referenced by versions","metric":"","query":"storage_stale_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by non-current versions","metric":"","query":"storage_old_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"referenced by current version","metric":"","query":"storage_current_version_object_count{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Object Total Number","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The number of bytes that have been written by compaction.Flush refers to the process of compacting Memtables to SSTables at Level 0.Compaction refers to the process of compacting SSTables at one level to another level.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":35,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Compaction - {{job}}","metric":"","query":"sum(storage_level_compact_write{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Flush - {{job}}","metric":"","query":"sum(compactor_write_build_l0_bytes{job=~\"$job\",instance=~\"$node\"}) by (job) > 0","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Write Bytes","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The remote storage read/write throughput","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"Bps"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":36,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"read - {{job}}","metric":"","query":"sum(rate(object_store_read_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"write - {{job}}","metric":"","query":"sum(rate(object_store_write_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (job)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Storage Remote I/O (Bytes/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Size statistics for checkpoint","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"bytes"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":16},"height":null,"hideTimeOverride":false,"id":37,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}}","metric":"","query":"histogram_quantile(0.5, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}}","metric":"","query":"histogram_quantile(0.99, sum(rate(state_store_sync_size_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"avg - {{job}}","metric":"","query":"sum by(le, job) (rate(state_store_sync_size_sum{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) / sum by(le, job) (rate(state_store_sync_size_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval]))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Checkpoint Size","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Storage","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":46},"height":null,"hideTimeOverride":false,"id":38,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":39,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_name}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_source_output_rows_counts{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of bytes read by each source per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"MB/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":40,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_id}}","metric":"","query":"(sum by (source_id)(rate(partition_input_bytes{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])))/(1000*1000)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Source Throughput(MB/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"The figure shows the number of rows written into each materialized executor actor per second.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":41,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"materialized view {{table_name}} table_id {{materialized_view_id}}","metric":"","query":"sum(rate(stream_executor_row_count{executor_identity=~\".*MaterializeExecutor.*\",job=~\"$job\",instance=~\"$node\"}[$__rate_interval]) * on(actor_id) group_left(materialized_view_id, table_name) (group(table_info{table_type=~\"MATERIALIZED_VIEW\",job=~\"$job\",instance=~\"$node\"}) by (actor_id, materialized_view_id, table_name))) by (materialized_view_id, table_name)","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Materialized View Throughput(rows/s)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"Total number of rows that have been read from the backfill operator used by MV on MV","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":42,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Read Snapshot - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_snapshot_read_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"Upstream - table_id={{table_id}} actor={{actor_id}} @ {{instance}}","metric":"","query":"rate(stream_backfill_upstream_output_row_count{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Backfill Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"We first record the total blocking duration(ns) of output buffer of each actor. It shows how much time it takes an actor to process a message, i.e. a barrier, a watermark or rows of data, on average. Then we divide this duration by 1 second and show it as a percentage.","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"percentunit"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":43,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"fragment {{fragment_id}}->{{downstream_fragment_id}}","metric":"","query":"avg(rate(stream_actor_output_buffer_blocking_duration_ns{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (fragment_id, downstream_fragment_id) / 1000000000","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Actor Output Blocking Time Ratio (Backpressure)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Streaming","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":47},"height":null,"hideTimeOverride":false,"id":44,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":45,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of running query in distributed execution mode","metric":"","query":"distributed_running_query_num{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Running query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":46,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of rejected query in distributed execution mode","metric":"","query":"distributed_rejected_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Rejected query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":""},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":8},"height":null,"hideTimeOverride":false,"id":47,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["last"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"The number of completed query in distributed execution mode","metric":"","query":"distributed_completed_query_counter{job=~\"$job\",instance=~\"$node\"}","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Completed query in distributed execution mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":8},"height":null,"hideTimeOverride":false,"id":48,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(distributed_query_latency_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Distributed Execution Mode","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":16},"height":null,"hideTimeOverride":false,"id":49,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p50 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.5, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p90 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.9, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""},{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"p99 - {{job}} @ {{instance}}","metric":"","query":"histogram_quantile(0.95, sum(rate(frontend_latency_local_execution_bucket{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])) by (le, job, instance))","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Query Latency in Local Execution Mode","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Batch","transformations":[],"transparent":false,"type":"row"},{"cacheTimeout":null,"collapsed":true,"datasource":null,"description":null,"editable":true,"error":false,"fieldConfig":{"defaults":{"thresholds":{"mode":"absolute","steps":[]}}},"gridPos":{"h":1,"w":24,"x":0,"y":48},"height":null,"hideTimeOverride":false,"id":50,"interval":null,"links":[],"maxDataPoints":100,"maxPerRow":null,"minSpan":null,"panels":[{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":0,"y":0},"height":null,"hideTimeOverride":false,"id":51,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"source={{source_type}} @ {{source_id}}","metric":"","query":"rate(connector_source_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Source Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"},{"cacheTimeout":null,"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"description":"","editable":true,"error":false,"fieldConfig":{"defaults":{"color":{"mode":"palette-classic"},"custom":{"axisLabel":"","axisPlacement":"auto","axisSoftMax":null,"axisSoftMin":null,"barAlignment":0,"drawStyle":"line","fillOpacity":10,"gradientMode":"none","hideFrom":{"legend":false,"tooltip":false,"viz":false},"lineInterpolation":"linear","lineWidth":1,"pointSize":5,"scaleDistribution":{"log":2,"type":"linear"},"showPoints":"auto","spanNulls":false,"stacking":{},"thresholdsStyle":{"mode":"off"}},"decimals":null,"mappings":[],"max":null,"min":null,"thresholds":{"mode":"absolute","steps":[]},"unit":"rows/s"},"overrides":[]},"gridPos":{"h":8,"w":12,"x":12,"y":0},"height":null,"hideTimeOverride":false,"id":52,"interval":"1s","links":[],"maxDataPoints":1000,"maxPerRow":null,"minSpan":null,"options":{"legend":{"calcs":["mean"],"displayMode":"table","placement":"bottom"},"tooltip":{"mode":"single","sort":"none"}},"repeat":null,"repeatDirection":null,"span":null,"targets":[{"datasource":{"type":"prometheus","uid":"risedev-prometheus"},"expr":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","format":"time_series","hide":false,"instant":false,"interval":"","intervalFactor":2,"legendFormat":"sink={{connector_type}} @ {{sink_id}}","metric":"","query":"rate(connector_sink_rows_received{job=~\"$job\",instance=~\"$node\"}[$__rate_interval])","refId":"","step":10,"target":""}],"timeFrom":null,"timeShift":null,"title":"Connector Sink Throughput(rows)","transformations":[],"transparent":false,"type":"timeseries"}],"repeat":null,"repeatDirection":null,"span":null,"targets":[],"timeFrom":null,"timeShift":null,"title":"Connector Node","transformations":[],"transparent":false,"type":"row"}],"refresh":"10s","rows":[],"schemaVersion":12,"sharedCrosshair":true,"style":"dark","tags":["risingwave"],"templating":{"list":[{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, instance)","description":"Reporting instance of the metric","hide":0,"includeAll":true,"label":"Node","multi":true,"name":"node","options":[],"query":{"query":"label_values(process_cpu_seconds_total, instance)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"},{"current":{"selected":false,"text":"All","value":"__all"},"definition":"label_values(process_cpu_seconds_total, job)","description":"Reporting job of the metric","hide":0,"includeAll":true,"label":"Job","multi":true,"name":"job","options":[],"query":{"query":"label_values(process_cpu_seconds_total, job)","refId":"StandardVariableQuery"},"refresh":2,"regex":"","skipUrlSync":false,"sort":6,"type":"query"}]},"time":{"from":"now-30m","to":"now"},"timepicker":{"hidden":false,"nowDelay":null,"refresh_intervals":["5s","10s","30s","1m","5m","15m","30m","1h","2h","1d"],"time_options":["5m","15m","1h","6h","12h","24h","2d","7d","30d"]},"timezone":"browser","title":"risingwave_dashboard","uid":"Fcy3uV1nz","version":0} diff --git a/integration_tests/cassandra-and-scylladb-sink/create_sink.sql b/integration_tests/cassandra-and-scylladb-sink/create_sink.sql index f3d0982a9bafe..a0a305aebd0e0 100644 --- a/integration_tests/cassandra-and-scylladb-sink/create_sink.sql +++ b/integration_tests/cassandra-and-scylladb-sink/create_sink.sql @@ -21,3 +21,27 @@ FROM cassandra.table = 'demo_bhv_table', cassandra.datacenter = 'datacenter1', ); + +CREATE SINK cassandra_types_sink +FROM + cassandra_types WITH ( + connector = 'cassandra', + type = 'append-only', + force_append_only='true', + cassandra.url = 'cassandra:9042', + cassandra.keyspace = 'demo', + cassandra.table = 'cassandra_types', + cassandra.datacenter = 'datacenter1', +); + +CREATE SINK scylladb_types_sink +FROM + cassandra_types WITH ( + connector = 'cassandra', + type = 'append-only', + force_append_only='true', + cassandra.url = 'scylladb:9042', + cassandra.keyspace = 'demo', + cassandra.table = 'cassandra_types', + cassandra.datacenter = 'datacenter1', +); diff --git a/integration_tests/cassandra-and-scylladb-sink/create_source.sql b/integration_tests/cassandra-and-scylladb-sink/create_source.sql index 292c3265a11e2..460e616aed358 100644 --- a/integration_tests/cassandra-and-scylladb-sink/create_source.sql +++ b/integration_tests/cassandra-and-scylladb-sink/create_source.sql @@ -16,3 +16,27 @@ CREATE table user_behaviors ( fields.user_name.length = '10', datagen.rows.per.second = '10' ) FORMAT PLAIN ENCODE JSON; + +CREATE TABLE cassandra_types ( + types_id int, + c_boolean boolean, + c_smallint smallint, + c_integer integer, + c_bigint bigint, + c_decimal decimal, + c_real real, + c_double_precision double precision, + c_varchar varchar, + c_bytea bytea, + c_date date, + c_time time, + c_timestamptz timestamptz, + c_interval interval, + PRIMARY KEY (types_id) +); + +INSERT INTO cassandra_types VALUES (1, False, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00.123456', '0001-01-01 00:00:00.123456'::timestamptz, '0 second'); + +INSERT INTO cassandra_types VALUES (2, False, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, 'aa', '\xff', '1970-01-01', '00:00:00', '1970-01-01 00:00:00Z', '4 hour'); + +INSERT INTO cassandra_types VALUES (3, True, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', '\xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', '9999-12-31', '23:59:59.999999', '9999-12-31 23:59:59.999999Z', '9990 year'); diff --git a/integration_tests/cassandra-and-scylladb-sink/prepare.sh b/integration_tests/cassandra-and-scylladb-sink/prepare.sh index 690537d878208..adba2a7cfba6e 100755 --- a/integration_tests/cassandra-and-scylladb-sink/prepare.sh +++ b/integration_tests/cassandra-and-scylladb-sink/prepare.sh @@ -3,7 +3,7 @@ set -euo pipefail # wait for cassandra and scylladb to start up -sleep 60 +sleep 30 # setup cassandra docker compose exec cassandra cqlsh -f prepare_cassandra_and_scylladb.sql diff --git a/integration_tests/cassandra-and-scylladb-sink/prepare_cassandra_and_scylladb.sql b/integration_tests/cassandra-and-scylladb-sink/prepare_cassandra_and_scylladb.sql index 1c221771c2e4c..b4906cd2dc582 100644 --- a/integration_tests/cassandra-and-scylladb-sink/prepare_cassandra_and_scylladb.sql +++ b/integration_tests/cassandra-and-scylladb-sink/prepare_cassandra_and_scylladb.sql @@ -5,3 +5,20 @@ CREATE table demo_bhv_table( target_id text, event_timestamp timestamp, ); + +CREATE table cassandra_types ( + types_id int primary key, + c_boolean boolean, + c_smallint smallint, + c_integer int, + c_bigint bigint, + c_decimal decimal, + c_real float, + c_double_precision double, + c_varchar text, + c_bytea blob, + c_date date, + c_time time, + c_timestamptz timestamp, + c_interval duration +); diff --git a/integration_tests/cassandra-and-scylladb-sink/sink_check.py b/integration_tests/cassandra-and-scylladb-sink/sink_check.py index 2087e002d9f44..d516d3a7c2de1 100644 --- a/integration_tests/cassandra-and-scylladb-sink/sink_check.py +++ b/integration_tests/cassandra-and-scylladb-sink/sink_check.py @@ -4,7 +4,7 @@ sleep(30) -relations = ['demo.demo_bhv_table'] +relations = ['demo.demo_bhv_table', 'demo.cassandra_types'] dbs = ['cassandra', 'scylladb'] failed_cases = [] diff --git a/integration_tests/client-library/client_test.py b/integration_tests/client-library/client_test.py index cd1eaa9304372..01473977201a5 100644 --- a/integration_tests/client-library/client_test.py +++ b/integration_tests/client-library/client_test.py @@ -4,48 +4,64 @@ def check_go(): - print("--- go client test") subprocess.run(["docker", "compose", "exec", "go-lang", "bash", "-c", "cd /go-client && ./run.sh"], check=True) def check_python(): - print("--- python client test") subprocess.run(["docker", "compose", "exec", "python", "bash", "-c", "cd /python-client && pip3 install -r requirements.txt && pytest"], check=True) def check_java(): - print("--- java client test") subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "apt-get update && apt-get install -y maven"], check=True) subprocess.run(["docker", "compose", "exec", "java", "bash", "-c", "cd /java-client && mvn clean test"], check=True) +def check_nodejs(): + subprocess.run( + ["docker", "compose", "exec", "nodejs", "bash", "-c", "cd /nodejs-client && npm install && npm test"], + check=True) + + +def check_php(): + subprocess.run( + ["docker", "compose", "exec", "php", "bash", "-c", "cd /php-client && phpunit tests/RWClientTest.php"], + check=True) + + +def check_ruby(): + subprocess.run(["docker", "compose", "exec", "ruby", "bash", "-c", "cd /ruby-client && ruby test/crud_test.rb"], + check=True) + + subprocess.run(["docker", "compose", "up", "-d"], check=True) sleep(10) failed_cases = [] -try: - check_go() -except Exception as e: - print(e) - failed_cases.append("go client failed") - -try: - check_python() -except Exception as e: - print(e) - failed_cases.append("python client failed") - -try: - check_java() -except Exception as e: - print(e) - failed_cases.append("java client failed") +for client in ['go', 'python', 'java', 'nodejs', 'php', 'ruby']: + print(f"--- {client} client test") + try: + if client == 'go': + check_go() + elif client == 'python': + check_python() + elif client == 'java': + check_java() + elif client == 'nodejs': + check_nodejs() + elif client == 'php': + check_php() + elif client == 'ruby': + check_ruby() + except Exception as e: + print(e) + failed_cases.append(f"{client} client failed") if len(failed_cases) != 0: print(f"--- client check failed for case\n{failed_cases}") sys.exit(1) +print("--- docker compose down") subprocess.run(["docker", "compose", "down", "--remove-orphans", "-v"], check=True) diff --git a/integration_tests/client-library/docker-compose.yml b/integration_tests/client-library/docker-compose.yml index 0bad2564d453d..fc6fbd6000ba6 100644 --- a/integration_tests/client-library/docker-compose.yml +++ b/integration_tests/client-library/docker-compose.yml @@ -37,6 +37,23 @@ services: command: tail -f /dev/null volumes: - ./java:/java-client + nodejs: + image: node:21.6.0-bullseye-slim + command: tail -f /dev/null + volumes: + - ./nodejs:/nodejs-client + php: + image: php-library + build: ./php + command: tail -f /dev/null + volumes: + - ./php:/php-client + ruby: + image: ruby-library + build: ./ruby + command: tail -f /dev/null + volumes: + - ./ruby:/ruby-client volumes: risingwave-standalone: diff --git a/integration_tests/client-library/nodejs/.gitignore b/integration_tests/client-library/nodejs/.gitignore new file mode 100644 index 0000000000000..504afef81fbad --- /dev/null +++ b/integration_tests/client-library/nodejs/.gitignore @@ -0,0 +1,2 @@ +node_modules/ +package-lock.json diff --git a/integration_tests/client-library/nodejs/index.js b/integration_tests/client-library/nodejs/index.js new file mode 100644 index 0000000000000..fca6b4aa54037 --- /dev/null +++ b/integration_tests/client-library/nodejs/index.js @@ -0,0 +1,29 @@ +import pg from 'pg'; + +export function rw_client() { + let types = pg.types + types.setTypeParser(types.builtins.TIMESTAMP, parseTimestamp) + types.setTypeParser(types.builtins.DATE, parseDate) + types.setTypeParser(types.builtins.INT8, parseBigint) + + return new pg.Client({ + host: 'risingwave-standalone', + port: 4566, + database: 'dev', + user: 'root', + password: '', + types: types, + }) +} + +function parseDate(val) { + return val === null ? null : val +} + +function parseTimestamp(val) { + return val === null ? null : val +} + +function parseBigint(val) { + return val === null ? null : BigInt(val) +} diff --git a/integration_tests/client-library/nodejs/package.json b/integration_tests/client-library/nodejs/package.json new file mode 100644 index 0000000000000..752b9f192bcd3 --- /dev/null +++ b/integration_tests/client-library/nodejs/package.json @@ -0,0 +1,20 @@ +{ + "name": "rw-nodejs-client-test", + "version": "1.0.0", + "description": "rw nodejs client test", + "main": "index.js", + "scripts": { + "test": "mocha" + }, + "author": "", + "license": "Apache-2.0", + "devDependencies": { + "mocha": "^10.2.0", + "chai": "^5.0.0" + }, + "dependencies": { + "pg": "^8.11.3", + "postgres-interval": "^4.0.1" + }, + "type": "module" +} diff --git a/integration_tests/client-library/nodejs/test/crud_test.js b/integration_tests/client-library/nodejs/test/crud_test.js new file mode 100644 index 0000000000000..f479ab7b4478c --- /dev/null +++ b/integration_tests/client-library/nodejs/test/crud_test.js @@ -0,0 +1,44 @@ +import {checkInsertedData, createTable, deleteDataByName, dropTable, insertData, updateSalaryData} from './util.js' + +export async function test_crud(client) { + await createTable(client) + + let name = "John Doe" + let age = 30 + let salary = BigInt(50000) + let tripIDs = ['12345', '67890'] + let fareData = { + initial_charge: 3.0, + subsequent_charge: 1.5, + surcharge: 0.5, + tolls: 2.0, + } + let deci = 3.14159 + let birthdate = "1993-05-15" + let starttime = "18:20:00" + let timest = "1993-05-15 00:00:00" + let timestz = new Date() + let timegap = "5 hours" + await insertData(client, name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) + await checkInsertedData(client, name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) + + // Insert data with null values + let nullName = "Null Person" + let nullAge = 0 + let nullSalary = BigInt(0) + let nullTripIDs = [] + let nullFareData = {} + let nullBirthDate = "0001-01-01" + let nullStarttime = "00:00:00" + let nullTimest = "0001-01-01 00:00:00" + let nullTimestz = new Date(0) + let nullTimegap = "0" + let nullDeci = 0.0 + await insertData(client, nullName, nullAge, nullSalary, nullTripIDs, nullBirthDate, nullDeci, nullFareData, nullStarttime, nullTimest, nullTimestz, nullTimegap) + + await checkInsertedData(client, nullName, nullAge, nullSalary, nullTripIDs, nullBirthDate, nullDeci, nullFareData, nullStarttime, nullTimest, nullTimestz, nullTimegap) + + await updateSalaryData(client, name, BigInt(60000)) + await deleteDataByName(client, name) + await dropTable(client) +} diff --git a/integration_tests/client-library/nodejs/test/test.js b/integration_tests/client-library/nodejs/test/test.js new file mode 100644 index 0000000000000..11d16eabc0f51 --- /dev/null +++ b/integration_tests/client-library/nodejs/test/test.js @@ -0,0 +1,18 @@ +import { rw_client } from '../index.js' +import { test_crud } from "./crud_test.js"; + +let client = rw_client() + +beforeEach(async function () { + await client.connect() +}) + +afterEach(async function () { + await client.end() +}) + +describe('nodejs client test', function () { + it('crud test', async function () { + await test_crud(client) + }) +}); diff --git a/integration_tests/client-library/nodejs/test/util.js b/integration_tests/client-library/nodejs/test/util.js new file mode 100644 index 0000000000000..ba17e326918ed --- /dev/null +++ b/integration_tests/client-library/nodejs/test/util.js @@ -0,0 +1,108 @@ +import * as chai from "chai"; + +const assert = chai.assert + +export async function createTable(client) { + let createTableQuery = ` + CREATE TABLE sample_table_nodejs + ( + name VARCHAR, + age INTEGER, + salary BIGINT, + trip_id VARCHAR[], + birthdate DATE, + deci DOUBLE PRECISION, + fare STRUCT < initial_charge DOUBLE PRECISION, + subsequent_charge DOUBLE PRECISION, + surcharge DOUBLE PRECISION, + tolls DOUBLE PRECISION >, + starttime TIME, + timest TIMESTAMP, + timestz TIMESTAMPTZ, + timegap INTERVAL + ) + `; + await client.query(createTableQuery) + console.log("Table created successfully.") +} + +export async function dropTable(client) { + let query = "drop table sample_table_nodejs;" + await client.query(query) + console.log("Table dropped successfully.") +} + +export async function insertData(client, name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) { + let insertDateQuery = { + text: ` + INSERT INTO sample_table_nodejs (name, age, salary, trip_id, birthdate, deci, fare, starttime, timest, + timestz, timegap) + VALUES ($1, $2, $3, $4, $5, $6, ROW($7, $8, $9, $10), $11, $12, $13, $14); + `, + values: [name, age, salary, tripIDs, birthdate, deci, fareData.initial_charge, fareData.subsequent_charge, fareData.surcharge, fareData.tolls, starttime, timest, timestz, timegap] + } + await client.query(insertDateQuery) + console.log("Data inserted successfully.") +} + +export async function checkInsertedData(client, name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) { + await client.query("FLUSH;") + + let query = "SELECT name, age, salary, trip_id, birthdate, deci, fare, starttime, timest, timestz, timegap FROM sample_table_nodejs WHERE name=$1" + let res = await client.query(query, [name]) + let row = res.rows[0] + + let retrievedName = row.name + assert.deepStrictEqual(retrievedName, name) + + let retrievedAge = row.age + assert.deepStrictEqual(retrievedAge, age) + + let retrievedSalary = row.salary + assert.deepStrictEqual(retrievedSalary, salary) + + let retrievedTripIDs = row.trip_id + assert.deepStrictEqual(retrievedTripIDs, tripIDs) + + let retrievedBirthdate = row.birthdate + assert.deepStrictEqual(retrievedBirthdate, birthdate) + + let retrievedDeci = row.deci + assert.deepStrictEqual(retrievedDeci, deci) + + let retrievedFareData = row.fare + + let retrievedStarttime = row.starttime + assert.deepStrictEqual(retrievedStarttime, starttime) + + let retrievedTimest = row.timest + assert.deepStrictEqual(retrievedTimest, timest) + + let retrievedTimestz = row.timestz + assert.deepStrictEqual(retrievedTimestz, timestz) + + let retrievedTimegap = row.timegap.toPostgres() + assert.deepStrictEqual(retrievedTimegap, timegap) + + console.log("Data checked successfully.") +} + +export async function updateSalaryData(client, name, salary) { + let query = ` + UPDATE sample_table_nodejs + SET salary=$1 + WHERE name = $2; + ` + await client.query(query, [salary, name]) + console.log("Data updated successfully.") +} + +export async function deleteDataByName(client, name) { + let query = ` + DELETE + FROM sample_table_nodejs + WHERE name = $1; + ` + await client.query(query, [name]) + console.log("Data deleted successfully.") +} diff --git a/integration_tests/client-library/php/Dockerfile b/integration_tests/client-library/php/Dockerfile new file mode 100644 index 0000000000000..ddf3826712599 --- /dev/null +++ b/integration_tests/client-library/php/Dockerfile @@ -0,0 +1,12 @@ +FROM php:cli-bullseye + +# install phpunit for test +RUN curl -fsSL -o /usr/local/bin/phpunit https://phar.phpunit.de/phpunit-10.phar && \ + chmod +x /usr/local/bin/phpunit + +# install pdo-pgsql extension +RUN apt-get update && \ + apt-get install -y libpq-dev && \ + docker-php-ext-install pdo_pgsql + +CMD ["php", "-a"] diff --git a/integration_tests/client-library/php/RWPDO.php b/integration_tests/client-library/php/RWPDO.php new file mode 100644 index 0000000000000..9b7fe6a1d9e3f --- /dev/null +++ b/integration_tests/client-library/php/RWPDO.php @@ -0,0 +1,9 @@ + diff --git a/integration_tests/client-library/php/tests/RWClientTest.php b/integration_tests/client-library/php/tests/RWClientTest.php new file mode 100644 index 0000000000000..996da241394d2 --- /dev/null +++ b/integration_tests/client-library/php/tests/RWClientTest.php @@ -0,0 +1,98 @@ + 3.0, + "subsequent_charge" => 1.5, + "surcharge" => 0.5, + "tolls" => 2.0, + ]; + $deci = 3.14159; + $birthdate = new DateTime("1993-01-02"); + $starttime = new DateTime("20:00:00"); + $timest = new DateTime(); + $timestz = new DateTime(timezone: new DateTimeZone('UTC')); + $timegap = DateInterval::createFromDateString('2 hours'); + insertData($rw, $name, $age, $salary, $tripIDs, $birthdate, $deci, $fareData, $starttime, $timest, $timestz, $timegap); + $this->checkData($rw, $name, $age, $salary, $tripIDs, $birthdate, $deci, $fareData, $starttime, $timest, $timestz, $timegap); + + // Insert data with null values + $nullName = "Null Person"; + $nullAge = 0; + $nullSalary = 0; + $nullTripIDs = []; + $nullFareData = []; + $nullBirthdate = new DateTime('0001-01-01'); + $nullStarttime = new DateTime('00:00:00'); + $nullTimest = new DateTime('0001-01-01 00:00:00'); + $nullTimestz = new DateTime('1970-01-01 00:00:00', timezone: new DateTimeZone('UTC')); + $nullTimegap = DateInterval::createFromDateString('0 seconds'); + $nullDeci = 0.0; + insertData($rw, $nullName, $nullAge, $nullSalary, $nullTripIDs, $nullBirthdate, $nullDeci, $nullFareData, $nullStarttime, $nullTimest, $nullTimestz, $nullTimegap); + $this->checkData($rw, $nullName, $nullAge, $nullSalary, $nullTripIDs, $nullBirthdate, $nullDeci, $nullFareData, $nullStarttime, $nullTimest, $nullTimestz, $nullTimegap); + + updateSalaryData($rw, $name, 60000); + deleteDataByName($rw, $name); + + dropTable($rw); + } + + function checkData(RWPDO &$rw, $name, int $age, int $salary, $tripIDs, DateTime $birthdate, $deci, $fareData, DateTime $starttime, DateTime $timest, DateTime $timestz, DateInterval $timegap) { + $rw->exec('FLUSH;'); + + $query = 'SELECT name, age, salary, trip_id, birthdate, deci, fare, starttime, timest, timestz, timegap FROM sample_table_php WHERE name=:name'; + $stmt = $rw->prepare($query); + $stmt->execute([':name' => $name]); + + while ($row = $stmt->fetch(PDO::FETCH_ASSOC)) { + $retrievedName = $row['name']; + $this->assertEquals($retrievedName, $name); + + $retrievedAge = $row['age']; + $this->assertEquals($retrievedAge, $age); + + $retrievedSalary = $row['salary']; + $this->assertEquals($retrievedSalary, $salary); + + $retrievedTripIDsStr = trim($row['trip_id'], '{}'); + $retrievedTripIDs = []; + if ($retrievedTripIDsStr !== '') { + $retrievedTripIDs = explode(',', $retrievedTripIDsStr); + } + $this->assertEquals($retrievedTripIDs, $tripIDs); + + $retrievedBirthdate = new DateTime($row['birthdate']); + $this->assertEquals($retrievedBirthdate, $birthdate); + + $retrievedDeci = (float)$row['deci']; + $this->assertEquals($retrievedDeci, $deci); + + $retrievedFareData = $row['fare']; + + $retrievedStarttime = new DateTime($row['starttime']); + $this->assertEquals($retrievedStarttime, $starttime); + + $retrievedTimest = new DateTime($row['timest']); + $this->assertEquals($retrievedTimest, $timest); + + $retrievedTimestz = new DateTime($row['timestz']); + $this->assertEquals($retrievedTimestz, $timestz); + + $retrievedTimegap = $row['timegap']; + $this->assertEquals($retrievedTimegap, $timegap->format('%H:%I:%S')); + } + print("Data checked successfully.\n"); + } +} + +?> diff --git a/integration_tests/client-library/php/util.php b/integration_tests/client-library/php/util.php new file mode 100644 index 0000000000000..5ce81db6d68f0 --- /dev/null +++ b/integration_tests/client-library/php/util.php @@ -0,0 +1,66 @@ +, + starttime TIME, + timest TIMESTAMP, + timestz TIMESTAMPTZ, + timegap INTERVAL +) +EOT; + $rw->exec($query); + print("Create Table successfully!\n"); +} + +function dropTable(RWPDO &$rw) { + $rw->exec('DROP TABLE sample_table_php;'); + print("Drop Table successfully!\n"); +} + +function insertData(RWPDO &$rw, $name, int $age, int $salary, $tripIDs, DateTime $birthdate, $deci, $fareData, DateTime $starttime, DateTime $timest, DateTime $timestz, DateInterval $timegap) { + $insertQuery = <<prepare($insertQuery); + $stmt->execute([$name, $age, $salary, + '{'.implode(',',$tripIDs).'}', + $birthdate->format('Y-m-d'), + $deci, + $fareData['initial_charge'], $fareData['subsequent_charge'], $fareData['surcharge'], $fareData['tolls'], + $starttime->format('H:i:s.u'), + $timest->format('Y-m-d H:i:s.u'), + $timestz->format('Y-m-d H:i:s.uP'), + $timegap->format('%H:%I:%S') + ]); + print("Data inserted successfully.\n"); +} + +function updateSalaryData(RWPDO &$rw, $name, $salary) { + $query = 'UPDATE sample_table_php SET salary=:salary WHERE name = :name;'; + $stmt = $rw->prepare($query); + $stmt->execute([':name' => $name, ':salary' => $salary]); + print("Data updated successfully.\n"); +} + +function deleteDataByName(RWPDO &$rw, $name) { + $query = 'DELETE FROM sample_table_php WHERE name = :name;'; + $stmt = $rw->prepare($query); + $stmt->execute([':name' => $name]); + print("Data deleted successfully.\n"); +} + +?> diff --git a/integration_tests/client-library/ruby/Dockerfile b/integration_tests/client-library/ruby/Dockerfile new file mode 100644 index 0000000000000..5283b5191068c --- /dev/null +++ b/integration_tests/client-library/ruby/Dockerfile @@ -0,0 +1,9 @@ +FROM ruby:3.2-slim-bullseye + +# install ruby-pg +RUN apt-get update && \ + apt-get install -y libpq-dev && \ + apt-get install -y build-essential && \ + gem install pg + +CMD ["irb"] diff --git a/integration_tests/client-library/ruby/rw_conn.rb b/integration_tests/client-library/ruby/rw_conn.rb new file mode 100644 index 0000000000000..0cb645db3ac75 --- /dev/null +++ b/integration_tests/client-library/ruby/rw_conn.rb @@ -0,0 +1,9 @@ +#!/usr/bin/env ruby +require 'pg' + +def get_rw_conn(host: 'localhost', port: 4566, options: '', tty: '', dbname: 'dev', user: 'root', password: '') + conn = PG.connect(host: host, port: port, options: options, tty: tty, dbname: dbname, user: user, password: password) + # https://github.com/risingwavelabs/risingwave/issues/14682 + # conn.type_map_for_results = PG::BasicTypeMapForResults.new(conn) + conn +end diff --git a/integration_tests/client-library/ruby/test/crud_test.rb b/integration_tests/client-library/ruby/test/crud_test.rb new file mode 100644 index 0000000000000..11e0e188500c9 --- /dev/null +++ b/integration_tests/client-library/ruby/test/crud_test.rb @@ -0,0 +1,98 @@ +#!/usr/bin/env ruby +require_relative '../rw_conn' +require_relative 'util' +require 'date' +require 'test/unit' + +class SQLTest < Test::Unit::TestCase + def setup + @conn = get_rw_conn(host: 'risingwave-standalone') + end + + def test_crud + create_table(@conn) + + name = 'John Doe' + age = 30 + salary = 50000 + tripIDs = ['12345', '67890'] + fareData = { + "initial_charge" => 3.0, + "subsequent_charge" => 1.5, + "surcharge" => 0.5, + "tolls" => 2.0, + } + deci = 2.14159 + birthdate = Date.parse('1993-01-02') + starttime = "20:00:00" + timest = DateTime.now() + timestz = DateTime.now() + timegap = '02:00:00' + + insert_data(@conn, name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) + check_data(name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) + + # Insert data with null values + nullName = "Null Person"; + nullAge = 0 + nullSalary = 0 + nullTripIDs = [] + nullFareData = {} + nullBirthdate = Date.parse('0001-01-01') + nullStarttime = '00:00:00' + nullTimest = DateTime.parse('0001-01-01 00:00:00') + nullTimestz = DateTime.parse('1970-01-01T00:00:00Z') + nullTimegap = '00:00:00'; + nullDeci = 0.0; + + insert_data(@conn, nullName, nullAge, nullSalary, nullTripIDs, nullBirthdate, nullDeci, nullFareData, nullStarttime, nullTimest, nullTimestz, nullTimegap) + check_data(nullName, nullAge, nullSalary, nullTripIDs, nullBirthdate, nullDeci, nullFareData, nullStarttime, nullTimest, nullTimestz, nullTimegap) + + update_data(@conn, name, 60000) + delete_data(@conn, name) + drop_table(@conn) + end + + def check_data(name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) + @conn.exec('FLUSH;') + + select_query = "SELECT name, age, salary, trip_id, birthdate, deci, fare, starttime, timest, timestz, timegap FROM sample_table_ruby WHERE name=$1" + + res = @conn.exec_params(select_query, [name]) + + res.each do |row| + retrievedName = row['name'] + assert_equal(name, retrievedName) + + retrievedAge = row['age'].to_i + assert_equal(age, retrievedAge) + + retrievedSalary = row['salary'].to_i + assert_equal(salary, retrievedSalary) + + retrievedTripIDs = row['trip_id'] + assert_equal('{' + tripIDs.join(',') + '}', retrievedTripIDs) + + retrievedBirthdate = row['birthdate'] + assert_equal(birthdate, Date.parse(retrievedBirthdate)) + + retrievedDeci = row['deci'] + assert_equal(deci, retrievedDeci.to_f) + + retrievedFareData = row['fare'] + + retrievedStarttime = row['starttime'] + assert_equal(starttime, retrievedStarttime) + + retrievedTimest = row['timest'] + assert_equal(timest.strftime('%Y-%m-%d %H:%M:%S%z'), DateTime.parse(retrievedTimest).strftime('%Y-%m-%d %H:%M:%S%z')) + + retrievedTimestz = row['timestz'] + assert_equal(timestz.strftime('%Y-%m-%dT%H:%M:%S%zZ'), DateTime.parse(retrievedTimestz).strftime('%Y-%m-%dT%H:%M:%S%zZ')) + + retrievedTimegap = row['timegap'] + assert_equal(timegap, retrievedTimegap) + end + puts "Data checked successfully." + end +end diff --git a/integration_tests/client-library/ruby/test/util.rb b/integration_tests/client-library/ruby/test/util.rb new file mode 100644 index 0000000000000..6b1b07e91d086 --- /dev/null +++ b/integration_tests/client-library/ruby/test/util.rb @@ -0,0 +1,65 @@ +#!/usr/bin/env ruby + +def create_table(conn) + query = <<-EOF + CREATE TABLE sample_table_ruby + ( + name VARCHAR, + age INTEGER, + salary BIGINT, + trip_id VARCHAR[], + birthdate DATE, + deci DOUBLE PRECISION, + fare STRUCT < initial_charge DOUBLE PRECISION, + subsequent_charge DOUBLE PRECISION, + surcharge DOUBLE PRECISION, + tolls DOUBLE PRECISION >, + starttime TIME, + timest TIMESTAMP, + timestz TIMESTAMPTZ, + timegap INTERVAL + ) + EOF + conn.exec(query) + puts "Table created successfully." +end + +def drop_table(conn) + conn.exec('DROP TABLE sample_table_ruby;') + puts "Table dropped successfully." +end + +def insert_data(conn, name, age, salary, tripIDs, birthdate, deci, fareData, starttime, timest, timestz, timegap) + insert_query = <<-EOF + INSERT INTO sample_table_ruby (name, age, salary, trip_id, birthdate, deci, fare, starttime, timest, timestz, timegap) + VALUES ($1, $2, $3, $4, $5, $6, ROW($7, $8, $9, $10), $11, $12, $13, $14); + EOF + conn.exec_params(insert_query, [name, age, salary, + "{#{tripIDs.join(',')}}", + birthdate, + deci, + fareData["initial_charge"], + fareData["subsequent_charge"], + fareData["surcharge"], + fareData["tolls"], + starttime, + timest.strftime('%Y-%m-%d %H:%M:%S'), + timestz, + timegap + ]) + + puts "Data inserted successfully." +end + +def update_data(conn, name, salary) + update_query = "UPDATE sample_table_ruby SET salary=$1 WHERE name=$2" + + conn.exec_params(update_query, [salary, name]) + puts "Data updated successfully." +end + +def delete_data(conn, name) + conn.exec_params('DELETE FROM sample_table_ruby WHERE name=$1;', [name]) + + puts "Data deletion successfully." +end diff --git a/integration_tests/datagen/gen/generator.go b/integration_tests/datagen/gen/generator.go index f84ffe3fcdea4..5cc7dfd2acefa 100644 --- a/integration_tests/datagen/gen/generator.go +++ b/integration_tests/datagen/gen/generator.go @@ -6,6 +6,7 @@ import ( "datagen/sink/kafka" "datagen/sink/kinesis" "datagen/sink/mysql" + "datagen/sink/nats" "datagen/sink/postgres" "datagen/sink/pulsar" "datagen/sink/s3" @@ -21,6 +22,7 @@ type GeneratorConfig struct { Pulsar pulsar.PulsarConfig Kinesis kinesis.KinesisConfig S3 s3.S3Config + Nats nats.NatsConfig // Whether to print the content of every event. PrintInsert bool diff --git a/integration_tests/datagen/go.mod b/integration_tests/datagen/go.mod index 9a77c3ae0056b..2cf89ae8d3509 100644 --- a/integration_tests/datagen/go.mod +++ b/integration_tests/datagen/go.mod @@ -10,6 +10,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 github.com/lib/pq v1.10.7 github.com/linkedin/goavro/v2 v2.9.8 + github.com/nats-io/nats.go v1.33.1 github.com/urfave/cli v1.22.10 go.uber.org/ratelimit v0.2.0 gonum.org/v1/gonum v0.12.0 @@ -28,7 +29,7 @@ require ( github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/danieljoos/wincred v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/eapache/go-resiliency v1.3.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 // indirect github.com/eapache/queue v1.1.0 // indirect @@ -48,10 +49,12 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d // indirect - github.com/klauspost/compress v1.15.11 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mtibben/percent v0.2.1 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pierrec/lz4/v4 v4.1.17 // indirect github.com/pkg/errors v0.9.1 // indirect @@ -67,12 +70,13 @@ require ( github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/stretchr/testify v1.8.0 // indirect go.uber.org/atomic v1.7.0 // indirect - golang.org/x/crypto v0.17.0 // indirect + golang.org/x/crypto v0.18.0 // indirect golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/text v0.14.0 // indirect google.golang.org/appengine v1.6.7 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/integration_tests/datagen/go.sum b/integration_tests/datagen/go.sum index dfdff515ee018..e35807572608b 100644 --- a/integration_tests/datagen/go.sum +++ b/integration_tests/datagen/go.sum @@ -103,8 +103,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimfeld/httptreemux v5.0.1+incompatible h1:Qj3gVcDNoOthBAqftuD596rm4wg/adLLz5xh5CmpiCA= github.com/dimfeld/httptreemux v5.0.1+incompatible/go.mod h1:rbUlSV+CCpv/SuqUTP/8Bk2O3LyUV436/yaRGkhP6Z0= -github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b h1:HBah4D48ypg3J7Np4N+HY/ZR76fx3HEUGxDU6Uk39oQ= github.com/dvsekhvalnov/jose2go v0.0.0-20200901110807-248326c1351b/go.mod h1:7BvyPhdbLxMXIYTFPLsyJRFMsKmOZnQmzh6Gb+uquuM= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eapache/go-resiliency v1.3.0 h1:RRL0nge+cWGlxXbUzJ7yMcq6w2XBEr19dCN6HECGaT0= github.com/eapache/go-resiliency v1.3.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= @@ -284,8 +285,8 @@ github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNr github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.10.8/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= -github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -325,6 +326,13 @@ github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= +github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -447,8 +455,8 @@ golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -618,12 +626,12 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -634,6 +642,7 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -804,6 +813,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/integration_tests/datagen/load_gen.go b/integration_tests/datagen/load_gen.go index 6a67500077b30..77735983fb6d1 100644 --- a/integration_tests/datagen/load_gen.go +++ b/integration_tests/datagen/load_gen.go @@ -15,6 +15,7 @@ import ( "datagen/sink/kafka" "datagen/sink/kinesis" "datagen/sink/mysql" + "datagen/sink/nats" "datagen/sink/postgres" "datagen/sink/pulsar" "datagen/sink/s3" @@ -39,6 +40,8 @@ func createSink(ctx context.Context, cfg gen.GeneratorConfig) (sink.Sink, error) return kinesis.OpenKinesisSink(cfg.Kinesis) } else if cfg.Sink == "s3" { return s3.OpenS3Sink(cfg.S3) + } else if cfg.Sink == "nats" { + return nats.OpenNatsSink(cfg.Nats) } else { return nil, fmt.Errorf("invalid sink type: %s", cfg.Sink) } diff --git a/integration_tests/datagen/main.go b/integration_tests/datagen/main.go index 9c4d35226980a..135e470ea204a 100644 --- a/integration_tests/datagen/main.go +++ b/integration_tests/datagen/main.go @@ -205,6 +205,28 @@ func main() { }, HelpName: "datagen s3", }, + { + Name: "nats", + Flags: []cli.Flag{ + cli.BoolFlag{ + Name: "jetstream", + Usage: "Whether to use JetStream", + Required: false, + Destination: &cfg.Nats.JetStream, + }, + cli.StringFlag{ + Name: "url", + Usage: "The URL of the NATS server", + Required: true, + Destination: &cfg.Nats.Url, + }, + }, + Action: func(c *cli.Context) error { + cfg.Sink = "nats" + return runCommand() + }, + HelpName: "datagen nats", + }, }, Flags: []cli.Flag{ cli.BoolFlag{ diff --git a/integration_tests/datagen/sink/nats/nats.go b/integration_tests/datagen/sink/nats/nats.go new file mode 100644 index 0000000000000..90df55a516fdf --- /dev/null +++ b/integration_tests/datagen/sink/nats/nats.go @@ -0,0 +1,76 @@ +package nats + +import ( + "context" + "datagen/sink" + "fmt" + "time" + + "github.com/nats-io/nats.go" + _ "github.com/nats-io/nats.go" + "github.com/nats-io/nats.go/jetstream" +) + +type NatsConfig struct { + Url string + JetStream bool +} + +type NatsSink struct { + nc *nats.Conn + config NatsConfig + js jetstream.JetStream +} + +func OpenNatsSink(config NatsConfig) (*NatsSink, error) { + nc, err := nats.Connect(config.Url) + if err != nil { + return nil, fmt.Errorf("failed to connect to NATS server: %v", err) + } + js, err := jetstream.New(nc) + if err != nil { + return nil, fmt.Errorf("failed to create JetStream instance: %v", err) + } + return &NatsSink{nc, config, js}, nil +} + +func (p *NatsSink) Prepare(topics []string) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if p.config.JetStream { + _, err := p.js.CreateOrUpdateStream(ctx, jetstream.StreamConfig{ + Name: "risingwave", + Subjects: topics, + }) + if err != nil { + return fmt.Errorf("failed to create JetStream stream: %v", err) + } + } + return nil +} + +func (p *NatsSink) Close() error { + p.nc.Close() + return nil +} + +func (p *NatsSink) WriteRecord(ctx context.Context, format string, record sink.SinkRecord) error { + data := sink.Encode(record, format) + + if p.config.JetStream { + _, err := p.js.Publish(ctx, record.Topic(), data) + if err != nil { + return fmt.Errorf("failed to publish record to JetStream: %v", err) + } + } else { + err := p.nc.Publish(record.Topic(), data) + if err != nil { + return fmt.Errorf("failed to request NATS server: %v", err) + } + } + return nil +} + +func (p *NatsSink) Flush(ctx context.Context) error { + return p.nc.Flush() +} diff --git a/integration_tests/debezium-mysql/create_source.sql b/integration_tests/debezium-mysql/create_source.sql index 6a15bf3b53850..cf7a6dc46f3af 100644 --- a/integration_tests/debezium-mysql/create_source.sql +++ b/integration_tests/debezium-mysql/create_source.sql @@ -3,4 +3,25 @@ CREATE TABLE orders (PRIMARY KEY(order_id)) with ( kafka.topic = 'mysql.mydb.orders', kafka.brokers = 'message_queue:29092', kafka.scan.startup.mode = 'earliest' -) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); \ No newline at end of file +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); + +CREATE TABLE mysql_all_data_types (PRIMARY KEY(id)) with ( + connector = 'kafka', + kafka.topic = 'mysql.mydb.mysql_all_data_types', + kafka.brokers = 'message_queue:29092', + kafka.scan.startup.mode = 'earliest' +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); + +CREATE TABLE mysql_types2 (PRIMARY KEY(id)) with ( + connector = 'kafka', + kafka.topic = 'mysql.mydb.mysql_types2', + kafka.brokers = 'message_queue:29092', + kafka.scan.startup.mode = 'earliest' +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); + +CREATE TABLE mysql_types3 (PRIMARY KEY(id)) with ( + connector = 'kafka', + kafka.topic = 'mysql.mydb.mysql_types3', + kafka.brokers = 'message_queue:29092', + kafka.scan.startup.mode = 'earliest' +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); diff --git a/integration_tests/debezium-mysql/data_check b/integration_tests/debezium-mysql/data_check index 1f4c99fed9650..5a32d4c80d293 100644 --- a/integration_tests/debezium-mysql/data_check +++ b/integration_tests/debezium-mysql/data_check @@ -1 +1 @@ -orders,product_count \ No newline at end of file +orders,product_count,mysql_all_data_types,mysql_types2,mysql_types3 diff --git a/integration_tests/debezium-mysql/docker-compose.yml b/integration_tests/debezium-mysql/docker-compose.yml index f3bc6b15b470e..77ff7689b78f8 100644 --- a/integration_tests/debezium-mysql/docker-compose.yml +++ b/integration_tests/debezium-mysql/docker-compose.yml @@ -37,6 +37,7 @@ services: volumes: - ./mysql/mysql.cnf:/etc/mysql/conf.d/mysql.cnf - ./mysql/mysql_bootstrap.sql:/docker-entrypoint-initdb.d/mysql_bootstrap.sql + - ./mysql_prepare.sql:/mysql_prepare.sql healthcheck: test: [ @@ -48,7 +49,8 @@ services: retries: 5 container_name: mysql debezium: - image: debezium/connect:1.9 + image: debezium-connect + build: . environment: BOOTSTRAP_SERVERS: message_queue:29092 GROUP_ID: 1 @@ -70,28 +72,7 @@ services: message_queue: { condition: service_healthy } mysql: { condition: service_healthy } container_name: debezium - debezium_deploy: - image: debezium/connect:1.9 - depends_on: - debezium: - condition: service_healthy - volumes: - - ./mysql/mysql_dbz.sh:/mysql_dbz.sh - entrypoint: [ bash, -c, /mysql_dbz.sh ] - container_name: debezium_deploy - restart: on-failure - datagen: - image: mysql:8.0 - depends_on: - mysql: { condition: service_healthy } - command: - - /bin/sh - - -c - - "mysql -p123456 -h mysql mydb < mysql_prepare.sql" - volumes: - - "./mysql_prepare.sql:/mysql_prepare.sql" - container_name: datagen - restart: on-failure + volumes: message_queue: external: false diff --git a/integration_tests/debezium-mysql/dockerfile b/integration_tests/debezium-mysql/dockerfile new file mode 100644 index 0000000000000..109d591b674f4 --- /dev/null +++ b/integration_tests/debezium-mysql/dockerfile @@ -0,0 +1,18 @@ +FROM quay.io/debezium/connect:2.5.0.Final + +ENV AVRO_PLUGION_DIR=$KAFKA_CONNECT_PLUGINS_DIR/confluent-avro-plugin + +ARG CONFLUENT_PLUGIN_VERSION=5.3.0 + +RUN mkdir $AVRO_PLUGION_DIR && cd $AVRO_PLUGION_DIR && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-converter/$CONFLUENT_PLUGIN_VERSION/kafka-connect-avro-converter-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-data/$CONFLUENT_PLUGIN_VERSION/kafka-connect-avro-data-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/common-config/$CONFLUENT_PLUGIN_VERSION/common-config-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/common-utils/$CONFLUENT_PLUGIN_VERSION/common-utils-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-schema-serializer/$CONFLUENT_PLUGIN_VERSION/kafka-schema-serializer-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-schema-registry-client/$CONFLUENT_PLUGIN_VERSION/kafka-schema-registry-client-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-avro-serializer/$CONFLUENT_PLUGIN_VERSION/kafka-avro-serializer-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://repo1.maven.org/maven2/com/google/guava/guava/31.0.1-jre/guava-31.0.1-jre.jar && \ + curl -sO https://repo1.maven.org/maven2/org/apache/avro/avro/1.8.2/avro-1.8.2.jar && \ + curl -sO https://repo1.maven.org/maven2/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar && \ + curl -sO https://repo1.maven.org/maven2/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar diff --git a/integration_tests/debezium-mysql/mysql/mysql_dbz.sh b/integration_tests/debezium-mysql/mysql/mysql_dbz.sh deleted file mode 100755 index 03477fcab49fd..0000000000000 --- a/integration_tests/debezium-mysql/mysql/mysql_dbz.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -echo "Deploying Debezium MySQL connector" - -curl -s -X PUT -H "Content-Type:application/json" http://debezium:8083/connectors/register-mysql/config \ - -d '{ - "connector.class": "io.debezium.connector.mysql.MySqlConnector", - "database.hostname": "mysql", - "database.port": 3306, - "database.user": "debezium", - "database.password": "mysqlpw", - "database.server.name": "mysql", - "database.server.id": "1", - "database.allowPublicKeyRetrieval": true, - "database.history.kafka.bootstrap.servers":"message_queue:29092", - "database.history.kafka.topic": "mysql-history", - "database.include.list": "mydb", - "time.precision.mode": "connect", - "include.schema.changes": false - }' diff --git a/integration_tests/debezium-mysql/mysql_prepare.sql b/integration_tests/debezium-mysql/mysql_prepare.sql index 65758e11f3af7..b327536fa7126 100644 --- a/integration_tests/debezium-mysql/mysql_prepare.sql +++ b/integration_tests/debezium-mysql/mysql_prepare.sql @@ -25,4 +25,58 @@ values 18.50, 2, 1 - ); \ No newline at end of file + ); + +CREATE TABLE mysql_types2 ( + id BIGINT PRIMARY KEY, + c_decimal DECIMAL, + c_date date, + c_time time, + c_datetime datetime +); + +INSERT INTO mysql_types2 VALUES (1, 0, '1001-01-01', '00:00:00', '1000-01-01 00:00:00.000000'); + +INSERT INTO mysql_types2 VALUES (2, -10.0, '1001-01-01', '-838:59:59.000000', '1000-01-01 00:00:00.000000'); + +INSERT INTO mysql_types2 VALUES (3, 10.12345, '9999-12-31', '20:59:59.000000', '9999-12-31 23:59:59.499999'); + +CREATE TABLE mysql_types3 ( + id BIGINT PRIMARY KEY, + c_decimal DECIMAL +); + +INSERT INTO mysql_types3 VALUES (1, 0); + +INSERT INTO mysql_types3 VALUES (2, -10.0); + +INSERT INTO mysql_types3 VALUES (3, 10.12345); + +CREATE TABLE mysql_all_data_types ( + id BIGINT PRIMARY KEY, + c_boolean boolean, + c_bit bit, + c_tinyint tinyint, + c_smallint smallint, + c_mediumint mediumint, + c_integer integer, + c_bigint bigint, + c_decimal decimal, + c_float float, + c_double double, + c_char_255 char(255), + c_varchar_10000 varchar(10000), + c_binary_255 binary(255), + c_varbinary_10000 varbinary(10000), + c_date date, + c_time time, + c_datetime datetime, + c_timestamp timestamp, + c_json JSON +); + +INSERT INTO mysql_all_data_types VALUES (1, False, 0, 0, 0, 0, 0, 0, 0, 0, 0, '', '', '', '', '1001-01-01', '00:00:00', '1000-01-01 00:00:00.000000', '1970-01-01 00:00:01', '{}'); + +INSERT INTO mysql_all_data_types VALUES (2, False, 0, -128, -32767, -8388608, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, '', '', 'aa', '', '1001-01-01', '23:59:59.000000', '1000-01-01 00:00:00.000000', '1970-01-01 00:00:01', '{"a": 1, "b":"c"}'); + +INSERT INTO mysql_all_data_types VALUES (3, True, 1, 127, 32767, 8388607, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', '9999-12-31', '20:59:59.000000', '9999-12-31 23:59:59.499999', '2038-01-19 03:14:07', '{"whatever":"meaningless"}'); diff --git a/integration_tests/debezium-mysql/prepare.sh b/integration_tests/debezium-mysql/prepare.sh new file mode 100755 index 0000000000000..b7f3af0c62f69 --- /dev/null +++ b/integration_tests/debezium-mysql/prepare.sh @@ -0,0 +1,72 @@ +#!/bin/bash + +set -euo pipefail + +# setup mysql +docker compose exec mysql bash -c "mysql -p123456 -h localhost -P 3306 mydb < mysql_prepare.sql" + +echo "Deploying Debezium MySQL connector" + +curl -S -X PUT -H "Content-Type:application/json" http://localhost:8083/connectors/mysql-default/config \ + -d '{ + "connector.class": "io.debezium.connector.mysql.MySqlConnector", + "topic.prefix": "mysql", + "database.hostname": "mysql", + "database.port": 3306, + "database.user": "debezium", + "database.password": "mysqlpw", + "database.server.id": "2304321", + "database.allowPublicKeyRetrieval": true, + "schema.history.internal.kafka.bootstrap.servers": "message_queue:29092", + "schema.history.internal.kafka.topic": "schema-changes.inventory", + "time.precision.mode": "adaptive_time_microseconds", + "decimal.handling.mode": "precise", + "binary.handling.mode": "bytes", + "include.schema.changes": false, + "table.include.list": "mydb.orders,mydb.mysql_all_data_types" + }' + +echo "Deploying Debezium MySQL connector" +# time: connect +# decimal: double +curl -S -X PUT -H "Content-Type:application/json" http://localhost:8083/connectors/mysql-connect-double/config \ + -d '{ + "connector.class": "io.debezium.connector.mysql.MySqlConnector", + "topic.prefix": "mysql", + "database.hostname": "mysql", + "database.port": 3306, + "database.user": "debezium", + "database.password": "mysqlpw", + "database.server.id": "412741", + "database.allowPublicKeyRetrieval": true, + "schema.history.internal.kafka.bootstrap.servers": "message_queue:29092", + "schema.history.internal.kafka.topic": "schema-changes.inventory", + "time.precision.mode": "connect", + "decimal.handling.mode": "double", + "include.schema.changes": false, + "table.include.list": "mydb.mysql_types2" + }' + +echo "Deploying Debezium MySQL connector" +# decimal: string +curl -S -X PUT -H "Content-Type:application/json" http://localhost:8083/connectors/mysql-string/config \ + -d '{ + "connector.class": "io.debezium.connector.mysql.MySqlConnector", + "topic.prefix": "mysql", + "database.hostname": "mysql", + "database.port": 3306, + "database.user": "debezium", + "database.password": "mysqlpw", + "database.server.id": "85092", + "database.allowPublicKeyRetrieval": true, + "schema.history.internal.kafka.bootstrap.servers": "message_queue:29092", + "schema.history.internal.kafka.topic": "schema-changes.inventory", + "decimal.handling.mode": "string", + "include.schema.changes": false, + "table.include.list": "mydb.mysql_types3" + }' + +echo 'sleep 3 minutes wait for debezium create all topics.' +sleep 180 + +echo 'Done' diff --git a/integration_tests/debezium-postgres/create_source.sql b/integration_tests/debezium-postgres/create_source.sql index 82564b0e33217..39a2a37156340 100644 --- a/integration_tests/debezium-postgres/create_source.sql +++ b/integration_tests/debezium-postgres/create_source.sql @@ -1,6 +1,27 @@ CREATE TABLE orders (PRIMARY KEY(order_id)) with ( connector = 'kafka', - kafka.topic = 'postgres.public.orders', + kafka.topic = 'pg.public.orders', kafka.brokers = 'message_queue:29092', kafka.scan.startup.mode = 'earliest' -) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); \ No newline at end of file +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); + +CREATE TABLE pg_all_data_types (PRIMARY KEY(id)) with ( + connector = 'kafka', + kafka.topic = 'pg.public.pg_all_data_types', + kafka.brokers = 'message_queue:29092', + kafka.scan.startup.mode = 'earliest' +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); + +CREATE TABLE pg_types2 (PRIMARY KEY(id)) with ( + connector = 'kafka', + kafka.topic = 'pg.public.pg_types2', + kafka.brokers = 'message_queue:29092', + kafka.scan.startup.mode = 'earliest' +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); + +CREATE TABLE pg_types3 (PRIMARY KEY(id)) with ( + connector = 'kafka', + kafka.topic = 'pg.public.pg_types3', + kafka.brokers = 'message_queue:29092', + kafka.scan.startup.mode = 'earliest' +) FORMAT DEBEZIUM ENCODE AVRO (schema.registry = 'http://message_queue:8081'); diff --git a/integration_tests/debezium-postgres/data_check b/integration_tests/debezium-postgres/data_check index 1f4c99fed9650..7c7403be35e56 100644 --- a/integration_tests/debezium-postgres/data_check +++ b/integration_tests/debezium-postgres/data_check @@ -1 +1 @@ -orders,product_count \ No newline at end of file +orders,product_count,pg_all_data_types,pg_types2,pg_types3 diff --git a/integration_tests/debezium-postgres/docker-compose.yml b/integration_tests/debezium-postgres/docker-compose.yml index 89748e45b44dd..ef9094e432f6f 100644 --- a/integration_tests/debezium-postgres/docker-compose.yml +++ b/integration_tests/debezium-postgres/docker-compose.yml @@ -27,7 +27,7 @@ services: service: message_queue postgres: - image: debezium/postgres:13 + image: debezium/postgres:16-alpine ports: - 5432:5432 environment: @@ -46,9 +46,11 @@ services: container_name: postgres volumes: - ./postgres/postgres_bootstrap.sql:/docker-entrypoint-initdb.d/postgres_bootstrap.sql + - "./postgres_prepare.sql:/postgres_prepare.sql" debezium: - image: debezium/connect:1.9 + image: debezium-connect + build: . environment: BOOTSTRAP_SERVERS: message_queue:29092 GROUP_ID: 1 @@ -69,40 +71,17 @@ services: postgres: { condition: service_healthy } container_name: debezium - debezium_deploy: - image: debezium/connect:1.9 - depends_on: - debezium: - condition: service_healthy - volumes: - - ./postgres/postgres_dbz.sh:/postgres_dbz.sh - entrypoint: [ bash, -c, /postgres_dbz.sh ] - container_name: debezium_deploy - restart: on-failure - - datagen: - image: postgres - depends_on: - postgres: { condition: service_healthy } - command: - - /bin/sh - - -c - - psql "postgresql://postgresuser:postgrespw@postgres:5432/mydb" -f ./postgres_prepare.sql - volumes: - - "./postgres_prepare.sql:/postgres_prepare.sql" - container_name: datagen - restart: on-failure - # Check out the connectors via 127.0.0.1:8000 - kafka-connect-ui: - image: landoop/kafka-connect-ui - ports: - - 8000:8000 - environment: - CONNECT_URL: http://debezium:8083 - container_name: kafka-connect-ui - depends_on: - message_queue: { condition: service_healthy } + # kafka-connect-ui: + # image: landoop/kafka-connect-ui + # platform: linux/amd64 + # ports: + # - 8000:8000 + # environment: + # CONNECT_URL: http://debezium:8083 + # container_name: kafka-connect-ui + # depends_on: + # message_queue: { condition: service_healthy } volumes: message_queue: diff --git a/integration_tests/debezium-postgres/dockerfile b/integration_tests/debezium-postgres/dockerfile new file mode 100644 index 0000000000000..109d591b674f4 --- /dev/null +++ b/integration_tests/debezium-postgres/dockerfile @@ -0,0 +1,18 @@ +FROM quay.io/debezium/connect:2.5.0.Final + +ENV AVRO_PLUGION_DIR=$KAFKA_CONNECT_PLUGINS_DIR/confluent-avro-plugin + +ARG CONFLUENT_PLUGIN_VERSION=5.3.0 + +RUN mkdir $AVRO_PLUGION_DIR && cd $AVRO_PLUGION_DIR && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-converter/$CONFLUENT_PLUGIN_VERSION/kafka-connect-avro-converter-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-connect-avro-data/$CONFLUENT_PLUGIN_VERSION/kafka-connect-avro-data-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/common-config/$CONFLUENT_PLUGIN_VERSION/common-config-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/common-utils/$CONFLUENT_PLUGIN_VERSION/common-utils-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-schema-serializer/$CONFLUENT_PLUGIN_VERSION/kafka-schema-serializer-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-schema-registry-client/$CONFLUENT_PLUGIN_VERSION/kafka-schema-registry-client-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://packages.confluent.io/maven/io/confluent/kafka-avro-serializer/$CONFLUENT_PLUGIN_VERSION/kafka-avro-serializer-$CONFLUENT_PLUGIN_VERSION.jar && \ + curl -sO https://repo1.maven.org/maven2/com/google/guava/guava/31.0.1-jre/guava-31.0.1-jre.jar && \ + curl -sO https://repo1.maven.org/maven2/org/apache/avro/avro/1.8.2/avro-1.8.2.jar && \ + curl -sO https://repo1.maven.org/maven2/org/codehaus/jackson/jackson-core-asl/1.9.13/jackson-core-asl-1.9.13.jar && \ + curl -sO https://repo1.maven.org/maven2/org/codehaus/jackson/jackson-mapper-asl/1.9.13/jackson-mapper-asl-1.9.13.jar diff --git a/integration_tests/debezium-postgres/postgres/postgres_dbz.sh b/integration_tests/debezium-postgres/postgres/postgres_dbz.sh deleted file mode 100755 index 817dda203ff5b..0000000000000 --- a/integration_tests/debezium-postgres/postgres/postgres_dbz.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env bash - -echo "Deploying Debezium Postgres connector" - -curl -s -X PUT -H "Content-Type: application/json" http://debezium:8083/connectors/register-postgres/config \ - -d '{ - "connector.class": "io.debezium.connector.postgresql.PostgresConnector", - "database.hostname": "postgres", - "database.port": 5432, - "database.user": "postgresuser", - "database.password": "postgrespw", - "database.dbname": "mydb", - "database.server.name": "postgres", - "database.schema": "public", - "database.history.kafka.bootstrap.servers": "message_queue:29092", - "database.history.kafka.topic": "postgres-history", - "time.precision.mode": "connect", - "include.schema.changes": false -}' diff --git a/integration_tests/debezium-postgres/postgres_prepare.sql b/integration_tests/debezium-postgres/postgres_prepare.sql index 249ed2fa6a75b..ef7dc4b68b6c8 100644 --- a/integration_tests/debezium-postgres/postgres_prepare.sql +++ b/integration_tests/debezium-postgres/postgres_prepare.sql @@ -29,4 +29,83 @@ INSERT INTO VALUES (1, 1558430840000, 'Bob', 10.50, 1, 1), (2, 1558430840001, 'Alice', 20.50, 2, 1), - (3, 1558430840002, 'Alice', 18.50, 2, 1); \ No newline at end of file + (3, 1558430840002, 'Alice', 18.50, 2, 1); + + +CREATE TABLE pg_types2 ( + id BIGINT PRIMARY KEY, + c_decimal DECIMAL, + c_interval INTERVAL, + c_date date, + c_time time, + c_timestamp timestamp +); + +ALTER TABLE + pg_types2 REPLICA IDENTITY FULL; + +INSERT INTO pg_types2 VALUES (1, 0, interval '0 second', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'); + +INSERT INTO pg_types2 VALUES (2, -108.12345678, '5 hours', '2024-01-01', '08:11:00.12345456', '2021-01-01 00:00:00.123456'); + +INSERT INTO pg_types2 VALUES (3, -10.0, '9990 year', '9999-12-31', '23:59:59', '9999-12-31 23:59:59'); + +CREATE TABLE pg_types3 ( + id BIGINT PRIMARY KEY, + c_decimal DECIMAL, + c_date date, + c_time time, + c_timestamp timestamp +); + +ALTER TABLE + pg_types3 REPLICA IDENTITY FULL; + +INSERT INTO pg_types3 VALUES (1, 0, '0001-01-01', '00:00:00', '0001-01-01 00:00:00'); + +INSERT INTO pg_types3 VALUES (2, -108.12345678, '2024-01-01', '08:11:00.12345456', '2021-01-01 00:00:00.123456'); + +INSERT INTO pg_types3 VALUES (3, -10.0, '9999-12-31', '23:59:59', '9999-12-31 23:59:59'); + + +CREATE TABLE pg_all_data_types ( + id BIGINT PRIMARY KEY, + c_boolean boolean, + c_smallint smallint, + c_integer integer, + c_bigint bigint, + c_decimal decimal, + c_real real, + c_double_precision double precision, + c_varchar varchar, + c_bytea bytea, + c_date date, + c_time time, + c_timestamp timestamp, + c_timestamptz timestamptz, + c_interval interval, + c_jsonb jsonb, + c_boolean_array boolean[], + c_smallint_array smallint[], + c_integer_array integer[], + c_bigint_array bigint[], + c_decimal_array decimal[], + c_real_array real[], + c_double_precision_array double precision[], + c_varchar_array varchar[], + c_bytea_array bytea[], + c_date_array date[], + c_time_array time[], + c_timestamp_array timestamp[], + c_timestamptz_array timestamptz[], + c_jsonb_array jsonb[] +); + +ALTER TABLE + pg_all_data_types REPLICA IDENTITY FULL; + +INSERT INTO pg_all_data_types VALUES (1, False, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00', '0001-01-01 00:00:00Z', interval '0 second', '{}', array[]::boolean[], array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::decimal[], array[]::real[], array[]::double precision[], array[]::varchar[], array[]::bytea[], array[]::date[], array[]::time[], array[]::timestamp[], array[]::timestamptz[], array[]::jsonb[]); + +INSERT INTO pg_all_data_types VALUES (2, False, -32767, -2147483647, -9223372036854775807, -108.12345678, -9999.999999, -10000.0, 'aa', '\xff', '2024-01-01', '08:11:00.12345456', '2021-01-01 00:00:00.123456', '1970-01-01 00:00:00.123456Z', '5 hours', '{"a": 1}', array[False::boolean]::boolean[], array[-32767::smallint]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[], array['\x00'::bytea]::bytea[], array['0001-01-01'::date]::date[], array['00:00:00'::time]::time[], array['0001-01-01 00:00:00'::timestamp::timestamp]::timestamp[], array['0001-01-01 00:00:00'::timestamptz::timestamptz]::timestamptz[], array['{}'::jsonb]::jsonb[]); + +INSERT INTO pg_all_data_types VALUES (3, True, 32767, 2147483647, 9223372036854775807, 1.1234567891, 9999.999999, 10000.0, 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', '\xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', '9999-12-31', '23:59:59', '9999-12-31 23:59:59'::timestamp, '9999-12-31 23:59:59'::timestamptz, interval '9990 year', '{"whatever":"meaningless"}', array[True::boolean]::boolean[], array[32767::smallint]::smallint[], array[2147483647::integer]::integer[], array[9223372036854775807::bigint]::bigint[], array[-10.0::decimal]::decimal[], array[9999.999999::real]::real[], array[10000.0::double precision]::double precision[], array['zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'::varchar]::varchar[], array['\xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff'::bytea]::bytea[], array['9999-12-31'::date]::date[], array['23:59:59'::time]::time[], array['9999-12-31 23:59:59'::timestamp::timestamp]::timestamp[], array['9999-12-31 23:59:59'::timestamptz::timestamptz]::timestamptz[], array['{"whatever":"meaningless"}'::jsonb]::jsonb[]); diff --git a/integration_tests/debezium-postgres/prepare.sh b/integration_tests/debezium-postgres/prepare.sh new file mode 100755 index 0000000000000..17d7227b0c1ee --- /dev/null +++ b/integration_tests/debezium-postgres/prepare.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +set -euo pipefail + +# setup postgres +docker compose exec postgres bash -c "psql postgresql://postgresuser:postgrespw@postgres:5432/mydb < postgres_prepare.sql" + +echo "Deploying Debezium Postgres connector" + +# default handling mode +curl -S -X PUT -H "Content-Type: application/json" http://localhost:8083/connectors/pg-default/config \ + -d '{ + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "pg", + "database.hostname": "postgres", + "database.port": 5432, + "database.user": "postgresuser", + "database.password": "postgrespw", + "database.dbname": "mydb", + "database.server.name": "postgres", + "database.schema": "public", + "database.history.kafka.bootstrap.servers": "message_queue:29092", + "database.history.kafka.topic": "postgres-history", + "time.precision.mode": "adaptive", + "decimal.handling.mode": "precise", + "interval.handling.mode": "numeric", + "include.schema.changes": false, + "slot.name": "debezium_1", + "table.include.list": "public.orders,public.pg_all_data_types" +}' + +echo "Deploying Debezium Postgres connector" +# time: adaptive_time_microseconds +# interval: string +# decimal: double +curl -S -X PUT -H "Content-Type: application/json" http://localhost:8083/connectors/pg-double-microseconds/config \ + -d '{ + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "pg", + "database.hostname": "postgres", + "database.port": 5432, + "database.user": "postgresuser", + "database.password": "postgrespw", + "database.dbname": "mydb", + "database.server.name": "postgres", + "database.schema": "public", + "database.history.kafka.bootstrap.servers": "message_queue:29092", + "database.history.kafka.topic": "postgres-history", + "time.precision.mode": "adaptive_time_microseconds", + "interval.handling.mode": "string", + "decimal.handling.mode": "double", + "include.schema.changes": false, + "slot.name": "debezium_2", + "table.include.list": "public.pg_types2" +}' + +echo "Deploying Debezium Postgres connector" +# time: connect +# decimal: string +curl -S -X PUT -H "Content-Type: application/json" http://localhost:8083/connectors/pg-connect-string/config \ + -d '{ + "connector.class": "io.debezium.connector.postgresql.PostgresConnector", + "topic.prefix": "pg", + "database.hostname": "postgres", + "database.port": 5432, + "database.user": "postgresuser", + "database.password": "postgrespw", + "database.dbname": "mydb", + "database.server.name": "postgres", + "database.schema": "public", + "database.history.kafka.bootstrap.servers": "message_queue:29092", + "database.history.kafka.topic": "postgres-history", + "time.precision.mode": "connect", + "decimal.handling.mode": "string", + "include.schema.changes": false, + "slot.name": "debezium_3", + "table.include.list": "public.pg_types3" +}' + +echo 'sleep two minutes wait for debezium create all topics.' +sleep 120 + +echo 'Done' diff --git a/integration_tests/doris-sink/README.md b/integration_tests/doris-sink/README.md index 75baa2d2449f1..b62c2d2e3adcf 100644 --- a/integration_tests/doris-sink/README.md +++ b/integration_tests/doris-sink/README.md @@ -45,16 +45,10 @@ GRANT ALL ON *.* TO 'users'@'%'; 4. Execute the SQL queries in sequence: -- append-only sql: - - create_source.sql - - create_mv.sql - - create_sink.sql - -- upsert sql: - - upsert/create_table.sql - - upsert/create_mv.sql - - upsert/create_sink.sql - - upsert/insert_update_delete.sql +- create_source.sql +- create_mv.sql +- create_sink.sql +- update_delete.sql We only support `upsert` with doris' `UNIQUE KEY` diff --git a/integration_tests/doris-sink/create_mv.sql b/integration_tests/doris-sink/create_mv.sql index c367e6f2baa94..6e466703b0769 100644 --- a/integration_tests/doris-sink/create_mv.sql +++ b/integration_tests/doris-sink/create_mv.sql @@ -5,3 +5,11 @@ SELECT event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local FROM user_behaviors; + +CREATE MATERIALIZED VIEW upsert_bhv_mv AS +SELECT + user_id, + target_id, + event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local +FROM + upsert_user_behaviors; diff --git a/integration_tests/doris-sink/create_sink.sql b/integration_tests/doris-sink/create_sink.sql index fa0cfddf7bf16..7cd1ac24857e9 100644 --- a/integration_tests/doris-sink/create_sink.sql +++ b/integration_tests/doris-sink/create_sink.sql @@ -9,4 +9,17 @@ FROM doris.database = 'demo', doris.table='demo_bhv_table', force_append_only='true' -); \ No newline at end of file +); + +CREATE SINK upsert_doris_sink +FROM + upsert_bhv_mv WITH ( + connector = 'doris', + type = 'upsert', + doris.url = 'http://fe:8030', + doris.user = 'users', + doris.password = '123456', + doris.database = 'demo', + doris.table='upsert_table', + primary_key = 'user_id' +); diff --git a/integration_tests/doris-sink/create_source.sql b/integration_tests/doris-sink/create_source.sql index ed7c02341638a..0e42308511121 100644 --- a/integration_tests/doris-sink/create_source.sql +++ b/integration_tests/doris-sink/create_source.sql @@ -14,3 +14,20 @@ CREATE table user_behaviors ( fields.user_id.end = '1000', datagen.rows.per.second = '100' ) FORMAT PLAIN ENCODE JSON; + +CREATE table upsert_user_behaviors ( + user_id int, + target_id VARCHAR, + target_type VARCHAR, + event_timestamp TIMESTAMPTZ, + behavior_type VARCHAR, + parent_target_type VARCHAR, + parent_target_id VARCHAR, + PRIMARY KEY(user_id) +); + +INSERT INTO upsert_user_behaviors VALUES + (1,'1','1','2020-01-01T01:01:01Z','1','1','1'), + (2,'2','2','2020-01-01T01:01:02Z','2','2','2'), + (3,'3','3','2020-01-01T01:01:03Z','3','3','3'), + (4,'4','4','2020-01-01T01:01:04Z','4','4','4'); diff --git a/integration_tests/doris-sink/docker-compose.yml b/integration_tests/doris-sink/docker-compose.yml index 74fecbee2baab..fc7cfd751e989 100644 --- a/integration_tests/doris-sink/docker-compose.yml +++ b/integration_tests/doris-sink/docker-compose.yml @@ -74,6 +74,15 @@ services: networks: mynetwork: ipv4_address: 172.21.0.9 + postgres: + image: postgres:latest + command: tail -f /dev/null + volumes: + - "./update_delete.sql:/update_delete.sql" + restart: on-failure + networks: + mynetwork: + ipv4_address: 172.21.0.11 volumes: risingwave-standalone: external: false diff --git a/integration_tests/doris-sink/doris_prepare.sql b/integration_tests/doris-sink/doris_prepare.sql index c95e8ac3f9b32..b65e419999caf 100644 --- a/integration_tests/doris-sink/doris_prepare.sql +++ b/integration_tests/doris-sink/doris_prepare.sql @@ -11,5 +11,15 @@ PROPERTIES ( "replication_allocation" = "tag.location.default: 1" ); +CREATE table upsert_table( + user_id int, + target_id text, + event_timestamp_local datetime +) UNIQUE KEY(`user_id`) +DISTRIBUTED BY HASH(`user_id`) BUCKETS 1 +PROPERTIES ( + "replication_allocation" = "tag.location.default: 1" +); + CREATE USER 'users'@'%' IDENTIFIED BY '123456'; GRANT ALL ON *.* TO 'users'@'%'; diff --git a/integration_tests/doris-sink/sink_check.py b/integration_tests/doris-sink/sink_check.py index 39109f4194fef..510cc867dcda4 100644 --- a/integration_tests/doris-sink/sink_check.py +++ b/integration_tests/doris-sink/sink_check.py @@ -1,7 +1,7 @@ import subprocess import sys -relations = ['demo.demo_bhv_table'] +relations = ['demo.demo_bhv_table', 'demo.upsert_table'] failed_cases = [] for rel in relations: @@ -18,6 +18,30 @@ if rows < 1: failed_cases.append(rel) +# update data +subprocess.run(["docker", "compose", "exec", "postgres", "bash", "-c", "psql -h risingwave-standalone -p 4566 -d dev -U root -f update_delete.sql"], check=True) + +# delete +sql = f"SELECT COUNT(*) FROM demo.upsert_table;" +command = f'mysql -uroot -P9030 -hfe -e "{sql}"' +output = subprocess.check_output( + ["docker", "compose", "exec", "mysql", "bash", "-c", command]) +rows = int(output.decode('utf-8').split('\n')[1]) +print(f"{rows} rows in demo.upsert_table") +if rows != 3: + print(f"rows expected 3, get {rows}") + failed_cases.append("delete demo.upsert_table") + +# update +sql = f"SELECT target_id FROM demo.upsert_table WHERE user_id = 3;" +command = f'mysql -uroot -P9030 -hfe -e "{sql}"' +output = subprocess.check_output( + ["docker", "compose", "exec", "mysql", "bash", "-c", command]) +id = int(output.decode('utf-8').split('\n')[1]) +if id != 30: + print(f"target_id expected 30, get {id}") + failed_cases.append("update demo.upsert_table") + if len(failed_cases) != 0: print(f"Data check failed for case {failed_cases}") sys.exit(1) diff --git a/integration_tests/doris-sink/update_delete.sql b/integration_tests/doris-sink/update_delete.sql new file mode 100644 index 0000000000000..adabd5163ef44 --- /dev/null +++ b/integration_tests/doris-sink/update_delete.sql @@ -0,0 +1,5 @@ +DELETE FROM upsert_user_behaviors WHERE user_id = 2; + +UPDATE upsert_user_behaviors SET target_id = 30 WHERE user_id = 3; + +FLUSH; diff --git a/integration_tests/doris-sink/upsert/create_mv.sql b/integration_tests/doris-sink/upsert/create_mv.sql deleted file mode 100644 index c367e6f2baa94..0000000000000 --- a/integration_tests/doris-sink/upsert/create_mv.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE MATERIALIZED VIEW bhv_mv AS -SELECT - user_id, - target_id, - event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local -FROM - user_behaviors; diff --git a/integration_tests/doris-sink/upsert/create_sink.sql b/integration_tests/doris-sink/upsert/create_sink.sql deleted file mode 100644 index e7bd5445ba557..0000000000000 --- a/integration_tests/doris-sink/upsert/create_sink.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE SINK bhv_doris_sink -FROM - bhv_mv WITH ( - connector = 'doris', - type = 'upsert', - doris.url = 'http://fe:8030', - doris.user = 'users', - doris.password = '123456', - doris.database = 'demo', - doris.table='demo_bhv_table', - primary_key = 'user_id' -); \ No newline at end of file diff --git a/integration_tests/doris-sink/upsert/create_table.sql b/integration_tests/doris-sink/upsert/create_table.sql deleted file mode 100644 index c6cfa87eed3c8..0000000000000 --- a/integration_tests/doris-sink/upsert/create_table.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE table user_behaviors ( - user_id int, - target_id VARCHAR, - target_type VARCHAR, - event_timestamp TIMESTAMPTZ, - behavior_type VARCHAR, - parent_target_type VARCHAR, - parent_target_id VARCHAR, - PRIMARY KEY(user_id) -); diff --git a/integration_tests/doris-sink/upsert/insert_update_delete.sql b/integration_tests/doris-sink/upsert/insert_update_delete.sql deleted file mode 100644 index f21353c161154..0000000000000 --- a/integration_tests/doris-sink/upsert/insert_update_delete.sql +++ /dev/null @@ -1,8 +0,0 @@ -INSERT INTO user_behaviors VALUES(1,'1','1','2020-01-01T01:01:01Z','1','1','1'), -(2,'2','2','2020-01-01T01:01:02Z','2','2','2'), -(3,'3','3','2020-01-01T01:01:03Z','3','3','3'), -(4,'4','4','2020-01-01T01:01:04Z','4','4','4'); - -DELETE FROM user_behaviors WHERE user_id = 2; - -UPDATE user_behaviors SET target_id = 30 WHERE user_id = 3; diff --git a/integration_tests/feature-store/server/Cargo.lock b/integration_tests/feature-store/server/Cargo.lock index 530adf76f0b76..03f652820a6bf 100644 --- a/integration_tests/feature-store/server/Cargo.lock +++ b/integration_tests/feature-store/server/Cargo.lock @@ -650,9 +650,9 @@ checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -660,7 +660,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 1.9.3", + "indexmap 2.0.0", "slab", "tokio", "tokio-util", @@ -993,9 +993,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", @@ -2251,9 +2251,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", @@ -2278,9 +2278,9 @@ dependencies = [ [[package]] name = "tonic-build" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d021fc044c18582b9a2408cd0dd05b1596e3ecdb5c4df822bb0183545683889" +checksum = "be4ef6dd70a610078cb4e338a0f79d06bc759ff1b22d2120c2ff02ae264ba9c2" dependencies = [ "prettyplease", "proc-macro2", @@ -2323,9 +2323,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "log", "pin-project-lite", diff --git a/integration_tests/feature-store/server/Cargo.toml b/integration_tests/feature-store/server/Cargo.toml index ce43a3f8af506..e84500f5f7923 100644 --- a/integration_tests/feature-store/server/Cargo.toml +++ b/integration_tests/feature-store/server/Cargo.toml @@ -11,17 +11,17 @@ edition = "2021" [dependencies] sqlx = { version = "0.7", features = ["runtime-tokio-native-tls", "postgres"] } tokio = { version = "1", features = ["full"] } -tonic = "0.10.2" +tonic = "0.11.0" reqwest = { version = "0.11", features = ["blocking"] } rdkafka = { version = "0.34", features = ["cmake-build"] } serde_json = "1.0" prost = "0.12" clap = "4.4.6" tokio-postgres = "0.7.10" -tonic-build = "0.10.2" +tonic-build = "0.11.0" [build-dependencies] -tonic-build = "0.10.2" +tonic-build = "0.11.0" [[bin]] name = "server" diff --git a/integration_tests/feature-store/server/model/requirements.txt b/integration_tests/feature-store/server/model/requirements.txt index 276361b0e1f83..95078ecfc43d7 100644 --- a/integration_tests/feature-store/server/model/requirements.txt +++ b/integration_tests/feature-store/server/model/requirements.txt @@ -1,4 +1,4 @@ -grpcio==1.53.0 +grpcio==1.53.2 numpy==1.24 protobuf==4.21.6 psycopg==3.0.16 diff --git a/integration_tests/feature-store/simulator/Cargo.lock b/integration_tests/feature-store/simulator/Cargo.lock index 7784c0040a515..9f03cf321b2ab 100644 --- a/integration_tests/feature-store/simulator/Cargo.lock +++ b/integration_tests/feature-store/simulator/Cargo.lock @@ -300,6 +300,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "errno" version = "0.3.5" @@ -454,9 +460,9 @@ checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" [[package]] name = "h2" -version = "0.3.21" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +checksum = "bb2c4422095b67ee78da96fbb51a4cc413b3b25883c7717ff7ca1ab31022c9c9" dependencies = [ "bytes", "fnv", @@ -464,7 +470,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 2.1.0", "slab", "tokio", "tokio-util", @@ -477,6 +483,12 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + [[package]] name = "hermit-abi" version = "0.3.3" @@ -583,7 +595,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "indexmap" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +dependencies = [ + "equivalent", + "hashbrown 0.14.3", ] [[package]] @@ -669,9 +691,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.8" +version = "0.8.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" dependencies = [ "libc", "wasi", @@ -1232,9 +1254,9 @@ dependencies = [ [[package]] name = "tonic" -version = "0.10.2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d560933a0de61cf715926b9cac824d4c883c2c43142f787595e48280c40a1d0e" +checksum = "76c4eb7a4e9ef9d4763600161f12f5070b92a578e1b634db88a6887844c91a13" dependencies = [ "async-stream", "async-trait", @@ -1265,7 +1287,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand", @@ -1291,9 +1313,9 @@ checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" [[package]] name = "tracing" -version = "0.1.39" +version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2ef2af84856a50c1d430afce2fdded0a4ec7eda868db86409b4543df0797f9" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ "pin-project-lite", "tracing-attributes", diff --git a/integration_tests/feature-store/simulator/Cargo.toml b/integration_tests/feature-store/simulator/Cargo.toml index 03264cde563bb..1058ac26ed55e 100644 --- a/integration_tests/feature-store/simulator/Cargo.toml +++ b/integration_tests/feature-store/simulator/Cargo.toml @@ -10,7 +10,7 @@ edition = "2021" [dependencies] tokio = { version = "1", features=["macros","rt", "rt-multi-thread"]} -tonic = "0.10.2" +tonic = "0.11.0" reqwest = { version = "0.11"} serde_json = "1.0" serde_derive = "1.0" diff --git a/integration_tests/http-sink/README.md b/integration_tests/http-sink/README.md new file mode 100644 index 0000000000000..d956cb4ea95a4 --- /dev/null +++ b/integration_tests/http-sink/README.md @@ -0,0 +1,34 @@ +# Demo: Sinking to Http + +In this demo, we want to showcase how RisingWave is able to sink data to Http. This feature is depended on https://github.com/getindata/flink-http-connector. + +It has a few limitations: +1. It offers only two options for HTTP method, i.e, PUT and POST. +2. It can only execute one request-reply round to the service (session-less). +3. It cannot handle status codes in the SQL API. + +Therefore, we suggest you to try Python UDF at first. + +### Demo: +1. Launch the cluster: + +```sh +docker-compose up -d +``` + +The cluster contains a RisingWave cluster and its necessary dependencies, a datagen that generates the data. + +2. Build an Http Server that can be built on its own + +3. Execute the SQL queries in sequence: + +- create_source.sql +- create_mv.sql +- create_sink.sql + +4. Check the contents in Http Server: +On the Http Server side it will receive the json string, something like: +``` +{"user_id":5,"target_id":"siFqrkdlCn"} +``` +The number of json is 1000 diff --git a/integration_tests/http-sink/create_mv.sql b/integration_tests/http-sink/create_mv.sql new file mode 100644 index 0000000000000..8a291a3c95ea7 --- /dev/null +++ b/integration_tests/http-sink/create_mv.sql @@ -0,0 +1,6 @@ +CREATE MATERIALIZED VIEW bhv_mv AS +SELECT + user_id, + target_id +FROM + user_behaviors; diff --git a/integration_tests/http-sink/create_sink.sql b/integration_tests/http-sink/create_sink.sql new file mode 100644 index 0000000000000..0644d1d51934b --- /dev/null +++ b/integration_tests/http-sink/create_sink.sql @@ -0,0 +1,11 @@ +CREATE sink bhv_http_sink FROM bhv_mv WITH ( + connector = 'http', + url = 'http://localhost:8080/endpoint', + format = 'json', + type = 'append-only', + force_append_only='true', + primary_key = 'user_id', + gid.connector.http.sink.header.Origin = '*', + "gid.connector.http.sink.header.X-Content-Type-Options" = 'nosniff', + "gid.connector.http.sink.header.Content-Type" = 'application/json' +); \ No newline at end of file diff --git a/integration_tests/http-sink/create_source.sql b/integration_tests/http-sink/create_source.sql new file mode 100644 index 0000000000000..c28c10f3616da --- /dev/null +++ b/integration_tests/http-sink/create_source.sql @@ -0,0 +1,18 @@ +CREATE table user_behaviors ( + user_id int, + target_id VARCHAR, + target_type VARCHAR, + event_timestamp TIMESTAMP, + behavior_type VARCHAR, + parent_target_type VARCHAR, + parent_target_id VARCHAR, + PRIMARY KEY(user_id) +) WITH ( + connector = 'datagen', + fields.user_id.kind = 'sequence', + fields.user_id.start = '1', + fields.user_id.end = '1000', + fields.user_name.kind = 'random', + fields.user_name.length = '10', + datagen.rows.per.second = '10' +) FORMAT PLAIN ENCODE JSON; \ No newline at end of file diff --git a/integration_tests/http-sink/docker-compose.yml b/integration_tests/http-sink/docker-compose.yml new file mode 100644 index 0000000000000..8fba5ff352dc0 --- /dev/null +++ b/integration_tests/http-sink/docker-compose.yml @@ -0,0 +1,37 @@ +--- +version: "3" +services: + risingwave-standalone: + extends: + file: ../../docker/docker-compose.yml + service: risingwave-standalone + etcd-0: + extends: + file: ../../docker/docker-compose.yml + service: etcd-0 + grafana-0: + extends: + file: ../../docker/docker-compose.yml + service: grafana-0 + minio-0: + extends: + file: ../../docker/docker-compose.yml + service: minio-0 + prometheus-0: + extends: + file: ../../docker/docker-compose.yml + service: prometheus-0 +volumes: + risingwave-standalone: + external: false + etcd-0: + external: false + grafana-0: + external: false + minio-0: + external: false + prometheus-0: + external: false + message_queue: + external: false +name: risingwave-compose diff --git a/integration_tests/iceberg-cdc/python/script/init.py b/integration_tests/iceberg-cdc/python/script/init.py index 289fa2f161889..b0745c248b25c 100644 --- a/integration_tests/iceberg-cdc/python/script/init.py +++ b/integration_tests/iceberg-cdc/python/script/init.py @@ -82,8 +82,11 @@ def init_risingwave_mv(args): s3.endpoint = 'http://minio-0:9301', s3.access.key = 'hummockadmin', s3.secret.key = 'hummockadmin', - database.name='demo', - table.name='s1.t1',warehouse.path = 's3://icebergdata/demo/s1/t1',s3.region = 'us-east-1' + catalog.name='demo', + database.name='s1', + table.name='t1', + warehouse.path = 's3://icebergdata/demo', + s3.region = 'us-east-1' ); """ ] diff --git a/integration_tests/iceberg-sink2/README.md b/integration_tests/iceberg-sink2/README.md index 54f34ea646e0e..496111fca71fe 100644 --- a/integration_tests/iceberg-sink2/README.md +++ b/integration_tests/iceberg-sink2/README.md @@ -1,7 +1,15 @@ -Use following steps to run: +# How to run the test -1. ./risedev d full-iceberg-bench -2. cd docker; docker compose up -d -3. poetry update -4. poetry run python init.py -5. poetry run python check.py \ No newline at end of file +Run following command to run the test: + +```bash +cd python +poetry update +poetry run python main.py +``` + +# How to override risingwave image version: + +```bash +export RW_IMAGE= +``` \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/docker-compose.yml b/integration_tests/iceberg-sink2/docker/docker-compose.yml deleted file mode 100644 index 566ce2895bf32..0000000000000 --- a/integration_tests/iceberg-sink2/docker/docker-compose.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: "3" - -services: - spark: - image: apache/spark:3.4.1 - container_name: spark - user: root - healthcheck: - test: netstat -ltn | grep -c ":15002" - interval: 5s - retries: 120 - ports: - - "15002:15002" - networks: - iceberg_net: - environment: - - SPARK_HOME=/opt/spark - - PYSPARK_PYTHON=/usr/bin/python3.9 - - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/spark/bin:/opt/spark/sbin - - AWS_ACCESS_KEY_ID=admin - - AWS_SECRET_ACCESS_KEY=password - - AWS_REGION=us-east-1 - volumes: - - './spark-script:/spark-script' - command: [ "bash", "/spark-script/spark-connect-server.sh" ] - -networks: - iceberg_net: diff --git a/integration_tests/iceberg-sink2/docker/hive/config.ini b/integration_tests/iceberg-sink2/docker/hive/config.ini new file mode 100644 index 0000000000000..d644f3c0d46a1 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/hive/config.ini @@ -0,0 +1,20 @@ +[risingwave] +db=dev +user=root +host=127.0.0.1 +port=4566 + +[sink] +connector = iceberg +type=append-only +force_append_only = true +catalog.type = hive +catalog.uri = thrift://metastore:9083 +warehouse.path = s3://icebergdata/demo +s3.endpoint=http://minio-0:9301 +s3.access.key = hummockadmin +s3.secret.key = hummockadmin +s3.region = ap-southeast-1 +catalog.name = demo +database.name=s1 +table.name=t1 \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/hive/docker-compose.yml b/integration_tests/iceberg-sink2/docker/hive/docker-compose.yml new file mode 100644 index 0000000000000..3314083c1077b --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/hive/docker-compose.yml @@ -0,0 +1,116 @@ +version: '3.8' + +services: + postgres: + image: postgres:16.1 + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: 123456 + POSTGRES_DB: metastore_db + expose: + - 5432 + ports: + - "5432:5432" + networks: + iceberg_net: + spark: + depends_on: + - minio-0 + - metastore + image: ghcr.io/icelake-io/icelake-spark:0.1 + environment: + - AWS_ACCESS_KEY_ID=hummockadmin + - AWS_SECRET_ACCESS_KEY=hummockadmin + - AWS_REGION=us-east-1 + - SPARK_HOME=/opt/spark + - PYSPARK_PYTHON=/usr/bin/python3.9 + - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/spark/bin:/opt/spark/sbin + user: root + networks: + iceberg_net: + links: + - minio-0:icebergdata.minio-0 + expose: + - 15002 + healthcheck: + test: netstat -ltn | grep -c 15002 + interval: 1s + retries: 1200 + volumes: + - ./spark-script:/spark-script + entrypoint: [ "/spark-script/spark-connect-server.sh" ] + + risingwave-standalone: + extends: + file: ../../../../docker/docker-compose.yml + service: risingwave-standalone + healthcheck: + test: + - CMD-SHELL + - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;' + interval: 1s + timeout: 30s + environment: + - AWS_REGION=us-east-1 + links: + - minio-0:icebergdata.minio-0 + networks: + iceberg_net: + + minio-0: + extends: + file: ../../../../docker/docker-compose.yml + service: minio-0 + entrypoint: " + /bin/sh -c ' + + set -e + + mkdir -p \"/data/icebergdata/demo\" + mkdir -p \"/data/hummock001\" + + /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\" + + '" + networks: + iceberg_net: + + etcd-0: + extends: + file: ../../../../docker/docker-compose.yml + service: etcd-0 + networks: + iceberg_net: + + metastore: + image: naushadh/hive-metastore + depends_on: + - postgres + environment: + - DATABASE_HOST=postgres + - DATABASE_DB=metastore_db + - DATABASE_USER=admin + - DATABASE_PASSWORD=123456 + - AWS_ACCESS_KEY_ID=hummockadmin + - AWS_SECRET_ACCESS_KEY=hummockadmin + - S3_ENDPOINT_URL=http://minio-0:9301 + - S3_BUCKET=icebergdata + - S3_PREFIX=demo + ports: + - "9083:9083" + expose: + - 9083 + networks: + iceberg_net: + +volumes: + risingwave-standalone: + external: false + etcd-0: + external: false + minio-0: + external: false + +networks: + iceberg_net: + name: iceberg \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/hive/spark-script/spark-connect-server.sh b/integration_tests/iceberg-sink2/docker/hive/spark-script/spark-connect-server.sh new file mode 100755 index 0000000000000..210a0663bea6e --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/hive/spark-script/spark-connect-server.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -ex + +JARS=$(find /opt/spark/deps -type f -name "*.jar" | tr '\n' ':') + +/opt/spark/sbin/start-connect-server.sh \ + --master local[3] \ + --driver-class-path $JARS \ + --conf spark.driver.bindAddress=0.0.0.0 \ + --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ + --conf spark.sql.catalog.demo.catalog-impl=org.apache.iceberg.hive.HiveCatalog \ + --conf spark.sql.catalog.demo.uri=thrift://metastore:9083 \ + --conf spark.sql.catalog.demo.clients=10 \ + --conf spark.sql.catalog.demo.warehouse=s3a://icebergdata/demo \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://minio-0:9301 \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.path.style.access=true \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=hummockadmin \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=hummockadmin \ + --conf spark.sql.defaultCatalog=demo + +tail -f /opt/spark/logs/spark*.out \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/jdbc/config.ini b/integration_tests/iceberg-sink2/docker/jdbc/config.ini new file mode 100644 index 0000000000000..a4fbd29bd3346 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/jdbc/config.ini @@ -0,0 +1,22 @@ +[risingwave] +db=dev +user=root +host=127.0.0.1 +port=4566 + +[sink] +connector = iceberg +type=append-only +force_append_only = true +warehouse.path = s3://icebergdata/demo +s3.endpoint=http://minio-0:9301 +s3.access.key = hummockadmin +s3.secret.key = hummockadmin +s3.region = ap-southeast-1 +catalog.name = demo +catalog.type = jdbc +catalog.uri = jdbc:postgresql://postgres:5432/iceberg +catalog.jdbc.user = admin +catalog.jdbc.password = 123456 +database.name=s1 +table.name=t1 \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/jdbc/docker-compose.yml b/integration_tests/iceberg-sink2/docker/jdbc/docker-compose.yml new file mode 100644 index 0000000000000..3f2bb75479563 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/jdbc/docker-compose.yml @@ -0,0 +1,96 @@ +version: '3.8' + +services: + postgres: + image: postgres:16.1 + environment: + POSTGRES_USER: admin + POSTGRES_PASSWORD: 123456 + POSTGRES_DB: iceberg + expose: + - 5432 + ports: + - "5432:5432" + networks: + iceberg_net: + + spark: + depends_on: + - minio-0 + - postgres + image: ghcr.io/icelake-io/icelake-spark:0.1 + environment: + - AWS_ACCESS_KEY_ID=hummockadmin + - AWS_SECRET_ACCESS_KEY=hummockadmin + - AWS_REGION=us-east-1 + - SPARK_HOME=/opt/spark + - PYSPARK_PYTHON=/usr/bin/python3.9 + - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/spark/bin:/opt/spark/sbin + user: root + networks: + iceberg_net: + links: + - minio-0:icebergdata.minio-0 + expose: + - 15002 + healthcheck: + test: netstat -ltn | grep -c 15002 + interval: 1s + retries: 1200 + volumes: + - ./spark-script:/spark-script + entrypoint: [ "/spark-script/spark-connect-server.sh" ] + + risingwave-standalone: + extends: + file: ../../../../docker/docker-compose.yml + service: risingwave-standalone + healthcheck: + test: + - CMD-SHELL + - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;' + interval: 1s + timeout: 30s + environment: + - AWS_REGION=us-east-1 + links: + - minio-0:icebergdata.minio-0 + networks: + iceberg_net: + + minio-0: + extends: + file: ../../../../docker/docker-compose.yml + service: minio-0 + entrypoint: " + /bin/sh -c ' + + set -e + + mkdir -p \"/data/icebergdata/demo\" + mkdir -p \"/data/hummock001\" + + /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\" + + '" + networks: + iceberg_net: + + etcd-0: + extends: + file: ../../../../docker/docker-compose.yml + service: etcd-0 + networks: + iceberg_net: + +volumes: + risingwave-standalone: + external: false + etcd-0: + external: false + minio-0: + external: false + +networks: + iceberg_net: + name: iceberg \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/jdbc/spark-script/spark-connect-server.sh b/integration_tests/iceberg-sink2/docker/jdbc/spark-script/spark-connect-server.sh new file mode 100755 index 0000000000000..8c3f752dc6414 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/jdbc/spark-script/spark-connect-server.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -ex + +JARS=$(find /opt/spark/deps -type f -name "*.jar" | tr '\n' ':') + +/opt/spark/sbin/start-connect-server.sh \ + --master local[3] \ + --driver-class-path $JARS \ + --conf spark.driver.bindAddress=0.0.0.0 \ + --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ + --conf spark.sql.catalog.demo.catalog-impl=org.apache.iceberg.jdbc.JdbcCatalog \ + --conf spark.sql.catalog.demo.io-impl=org.apache.iceberg.aws.s3.S3FileIO \ + --conf spark.sql.catalog.demo.warehouse=s3://icebergdata/demo \ + --conf spark.sql.catalog.demo.uri=jdbc:postgresql://postgres:5432/iceberg \ + --conf spark.sql.catalog.demo.jdbc.user=admin \ + --conf spark.sql.catalog.demo.jdbc.password=123456 \ + --conf spark.sql.catalog.demo.s3.endpoint=http://minio-0:9301 \ + --conf spark.sql.catalog.demo.s3.path.style.access=true \ + --conf spark.sql.catalog.demo.s3.access.key=hummockadmin \ + --conf spark.sql.catalog.demo.s3.secret.key=hummockadmin \ + --conf spark.sql.defaultCatalog=demo + +tail -f /opt/spark/logs/spark*.out \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/rest/config.ini b/integration_tests/iceberg-sink2/docker/rest/config.ini new file mode 100644 index 0000000000000..28b1e4c159f5b --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/rest/config.ini @@ -0,0 +1,20 @@ +[risingwave] +db=dev +user=root +host=127.0.0.1 +port=4566 + +[sink] +connector=iceberg +type=append-only +force_append_only=true +s3.endpoint=http://minio-0:9301 +s3.access.key = hummockadmin +s3.secret.key = hummockadmin +s3.region = ap-southeast-1 +catalog.type=rest +catalog.name=demo +catalog.uri = http://rest:8181 +warehouse.path = s3://icebergdata/demo +database.name=s1 +table.name=t1 \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/rest/docker-compose.yml b/integration_tests/iceberg-sink2/docker/rest/docker-compose.yml new file mode 100644 index 0000000000000..025db74c23cae --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/rest/docker-compose.yml @@ -0,0 +1,99 @@ +version: '3.8' + +services: + rest: + image: tabulario/iceberg-rest:0.6.0 + environment: + - AWS_ACCESS_KEY_ID=hummockadmin + - AWS_SECRET_ACCESS_KEY=hummockadmin + - AWS_REGION=us-east-1 + - CATALOG_CATOLOG__IMPL=org.apache.iceberg.jdbc.JdbcCatalog + - CATALOG_URI=jdbc:sqlite:file:/tmp/iceberg_rest_mode=memory + - CATALOG_WAREHOUSE=s3://icebergdata/demo + - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO + - CATALOG_S3_ENDPOINT=http://minio-0:9301 + depends_on: + - minio-0 + networks: + - iceberg_net + links: + - minio-0:icebergdata.minio-0 + expose: + - 8181 + + spark: + depends_on: + - minio-0 + - rest + image: ghcr.io/icelake-io/icelake-spark:0.1 + environment: + - AWS_ACCESS_KEY_ID=hummockadmin + - AWS_SECRET_ACCESS_KEY=hummockadmin + - AWS_REGION=us-east-1 + - SPARK_HOME=/opt/spark + - PYSPARK_PYTHON=/usr/bin/python3.9 + - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/spark/bin:/opt/spark/sbin + user: root + networks: + iceberg_net: + links: + - minio-0:icebergdata.minio-0 + expose: + - 15002 + healthcheck: + test: netstat -ltn | grep -c 15002 + interval: 1s + retries: 1200 + volumes: + - ./spark-script:/spark-script + entrypoint: ["/spark-script/spark-connect-server.sh"] + + risingwave-standalone: + extends: + file: ../../../../docker/docker-compose.yml + service: risingwave-standalone + healthcheck: + test: + - CMD-SHELL + - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;' + interval: 1s + timeout: 30s + networks: + iceberg_net: + + minio-0: + extends: + file: ../../../../docker/docker-compose.yml + service: minio-0 + entrypoint: " + + /bin/sh -c ' + + set -e + + mkdir -p \"/data/icebergdata/demo\" + mkdir -p \"/data/hummock001\" + + /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\" + + '" + networks: + iceberg_net: + + etcd-0: + extends: + file: ../../../../docker/docker-compose.yml + service: etcd-0 + networks: + iceberg_net: + +volumes: + risingwave-standalone: + external: false + etcd-0: + external: false + minio-0: + external: false + +networks: + iceberg_net: \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/rest/spark-script/spark-connect-server.sh b/integration_tests/iceberg-sink2/docker/rest/spark-script/spark-connect-server.sh new file mode 100755 index 0000000000000..7e79ae000a062 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/rest/spark-script/spark-connect-server.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -ex + +JARS=$(find /opt/spark/deps -type f -name "*.jar" | tr '\n' ':') + +/opt/spark/sbin/start-connect-server.sh \ + --master local[3] \ + --driver-class-path $JARS \ + --conf spark.driver.bindAddress=0.0.0.0 \ + --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ + --conf spark.sql.catalog.demo.catalog-impl=org.apache.iceberg.rest.RESTCatalog \ + --conf spark.sql.catalog.demo.uri=http://rest:8181 \ + --conf spark.sql.catalog.demo.s3.endpoint=http://minio-0:9301 \ + --conf spark.sql.catalog.demo.s3.path.style.access=true \ + --conf spark.sql.catalog.demo.s3.access.key=hummockadmin \ + --conf spark.sql.catalog.demo.s3.secret.key=hummockadmin \ + --conf spark.sql.defaultCatalog=demo + +tail -f /opt/spark/logs/spark*.out \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/spark-script/.gitignore b/integration_tests/iceberg-sink2/docker/spark-script/.gitignore deleted file mode 100644 index 51dcf07222856..0000000000000 --- a/integration_tests/iceberg-sink2/docker/spark-script/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -derby.log -metastore_db -.ivy \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/spark-script/init-table.sql b/integration_tests/iceberg-sink2/docker/spark-script/init-table.sql deleted file mode 100644 index 8f605921ce9e7..0000000000000 --- a/integration_tests/iceberg-sink2/docker/spark-script/init-table.sql +++ /dev/null @@ -1,18 +0,0 @@ -CREATE SCHEMA IF NOT EXISTS s1; - -USE s1; - -DROP TABLE IF EXISTS t1; - -CREATE TABLE t1 -( - id bigint, - name string, - distance bigint -) USING iceberg -TBLPROPERTIES ('format-version'='2'); - -INSERT INTO t1 VALUES (1, "a", 100), (2, "b", 200); - - - diff --git a/integration_tests/iceberg-sink2/docker/spark-script/insert-table.sql b/integration_tests/iceberg-sink2/docker/spark-script/insert-table.sql deleted file mode 100644 index ebeb17d270941..0000000000000 --- a/integration_tests/iceberg-sink2/docker/spark-script/insert-table.sql +++ /dev/null @@ -1,6 +0,0 @@ -USE s1; - -INSERT INTO t1 VALUES (3, "a", 300), (4, "b", 400); - - - diff --git a/integration_tests/iceberg-sink2/docker/spark-script/inspect-table.sql b/integration_tests/iceberg-sink2/docker/spark-script/inspect-table.sql deleted file mode 100644 index b626a74f4e53e..0000000000000 --- a/integration_tests/iceberg-sink2/docker/spark-script/inspect-table.sql +++ /dev/null @@ -1,7 +0,0 @@ -DESCRIBE demo.s1.t1.files; - -SELECT * FROM demo.s1.t1.files; - -DESCRIBE demo.s1.t1.manifests; - -SELECT * FROM demo.s1.t1.manifests; diff --git a/integration_tests/iceberg-sink2/docker/spark-script/query-table.sql b/integration_tests/iceberg-sink2/docker/spark-script/query-table.sql deleted file mode 100644 index 7de932bec5d49..0000000000000 --- a/integration_tests/iceberg-sink2/docker/spark-script/query-table.sql +++ /dev/null @@ -1 +0,0 @@ -SELECT * FROM demo.s1.t1 ORDER BY id ASC; diff --git a/integration_tests/iceberg-sink2/docker/spark-script/run-sql-file.sh b/integration_tests/iceberg-sink2/docker/spark-script/run-sql-file.sh deleted file mode 100644 index 685f228e4e016..0000000000000 --- a/integration_tests/iceberg-sink2/docker/spark-script/run-sql-file.sh +++ /dev/null @@ -1,27 +0,0 @@ -set -ex - -ICEBERG_VERSION=1.3.1 -DEPENDENCIES="org.apache.iceberg:iceberg-spark-runtime-3.3_2.12:$ICEBERG_VERSION,org.apache.hadoop:hadoop-aws:3.3.2" - -## add AWS dependency -#AWS_SDK_VERSION=2.20.18 -#AWS_MAVEN_GROUP=software.amazon.awssdk -#AWS_PACKAGES=( -# "bundle" -#) -#for pkg in "${AWS_PACKAGES[@]}"; do -# DEPENDENCIES+=",$AWS_MAVEN_GROUP:$pkg:$AWS_SDK_VERSION" -#done - -spark-sql --packages $DEPENDENCIES \ - --master local[3] \ - --files /spark-script/log4j.properties \ - --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \ - --conf spark.sql.catalog.demo.type=hadoop \ - --conf spark.sql.catalog.demo.warehouse=s3a://icebergdata/demo \ - --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://minio:9000 \ - --conf spark.sql.catalog.demo.hadoop.fs.s3a.path.style.access=true \ - --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=admin \ - --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=password \ - --conf spark.sql.defaultCatalog=demo \ - -f /spark-script/$1.sql diff --git a/integration_tests/iceberg-sink2/docker/spark-script/spark-connect-server.sh b/integration_tests/iceberg-sink2/docker/spark-script/spark-connect-server.sh deleted file mode 100644 index 2ca7e5026fea5..0000000000000 --- a/integration_tests/iceberg-sink2/docker/spark-script/spark-connect-server.sh +++ /dev/null @@ -1,20 +0,0 @@ -set -ex - -ICEBERG_VERSION=1.3.1 -SPARK_VERSION=3.4.1 - -PACKAGES="org.apache.iceberg:iceberg-spark-runtime-3.4_2.12:$ICEBERG_VERSION,org.apache.hadoop:hadoop-aws:3.3.2" -PACKAGES="$PACKAGES,org.apache.spark:spark-connect_2.12:$SPARK_VERSION" - -/opt/spark/sbin/start-connect-server.sh --packages $PACKAGES \ - --master local[3] \ - --conf spark.driver.bindAddress=0.0.0.0 \ - --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \ - --conf spark.sql.catalog.demo.type=hadoop \ - --conf spark.sql.catalog.demo.warehouse=s3a://renjie-iceberg-bench/wh \ - --conf spark.sql.catalog.demo.hadoop.fs.s3a.path.style.access=true \ - --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=admin \ - --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=password \ - --conf spark.sql.defaultCatalog=demo - -tail -f /opt/spark/logs/spark*.out \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/storage/config.ini b/integration_tests/iceberg-sink2/docker/storage/config.ini new file mode 100644 index 0000000000000..13e912b8fc3b8 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/storage/config.ini @@ -0,0 +1,19 @@ +[risingwave] +db=dev +user=root +host=127.0.0.1 +port=4566 + +[sink] +connector = iceberg +type=append-only +force_append_only = true +s3.endpoint=http://minio-0:9301 +s3.access.key = hummockadmin +s3.secret.key = hummockadmin +s3.region = ap-southeast-1 +catalog.type = storage +catalog.name = demo +warehouse.path = s3://icebergdata/demo +database.name=s1 +table.name=t1 \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/storage/docker-compose.yml b/integration_tests/iceberg-sink2/docker/storage/docker-compose.yml new file mode 100644 index 0000000000000..8d6d6f72d53c5 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/storage/docker-compose.yml @@ -0,0 +1,75 @@ +version: '3.8' + +services: + spark: + depends_on: + - minio-0 + image: ghcr.io/icelake-io/icelake-spark:0.1 + environment: + - SPARK_HOME=/opt/spark + - PYSPARK_PYTHON=/usr/bin/python3.9 + - PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/spark/bin:/opt/spark/sbin + user: root + networks: + iceberg_net: + links: + - minio-0:icebergdata.minio-0 + expose: + - 15002 + healthcheck: + test: netstat -ltn | grep -c 15002 + interval: 1s + retries: 1200 + volumes: + - ./spark-script:/spark-script + entrypoint: ["/spark-script/spark-connect-server.sh"] + + risingwave-standalone: + extends: + file: ../../../../docker/docker-compose.yml + service: risingwave-standalone + healthcheck: + test: + - CMD-SHELL + - bash -c 'printf \"GET / HTTP/1.1\n\n\" > /dev/tcp/127.0.0.1/4566; exit $$?;' + interval: 1s + timeout: 30s + networks: + iceberg_net: + + minio-0: + extends: + file: ../../../../docker/docker-compose.yml + service: minio-0 + entrypoint: " + + /bin/sh -c ' + + set -e + + mkdir -p \"/data/icebergdata/demo\" + mkdir -p \"/data/hummock001\" + + /usr/bin/docker-entrypoint.sh \"$$0\" \"$$@\" + + '" + networks: + iceberg_net: + + etcd-0: + extends: + file: ../../../../docker/docker-compose.yml + service: etcd-0 + networks: + iceberg_net: + +volumes: + risingwave-standalone: + external: false + etcd-0: + external: false + minio-0: + external: false + +networks: + iceberg_net: \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/docker/storage/spark-script/spark-connect-server.sh b/integration_tests/iceberg-sink2/docker/storage/spark-script/spark-connect-server.sh new file mode 100755 index 0000000000000..d37ed983fc236 --- /dev/null +++ b/integration_tests/iceberg-sink2/docker/storage/spark-script/spark-connect-server.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -ex + +JARS=$(find /opt/spark/deps -type f -name "*.jar" | tr '\n' ':') + +/opt/spark/sbin/start-connect-server.sh \ + --master local[3] \ + --driver-class-path $JARS \ + --conf spark.driver.bindAddress=0.0.0.0 \ + --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions \ + --conf spark.sql.catalog.demo=org.apache.iceberg.spark.SparkCatalog \ + --conf spark.sql.catalog.demo.type=hadoop \ + --conf spark.sql.catalog.demo.warehouse=s3a://icebergdata/demo \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.endpoint=http://minio-0:9301 \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.path.style.access=true \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.access.key=hummockadmin \ + --conf spark.sql.catalog.demo.hadoop.fs.s3a.secret.key=hummockadmin \ + --conf spark.sql.defaultCatalog=demo + +tail -f /opt/spark/logs/spark*.out \ No newline at end of file diff --git a/integration_tests/iceberg-sink2/python/check.py b/integration_tests/iceberg-sink2/python/check.py deleted file mode 100644 index 24d931da23601..0000000000000 --- a/integration_tests/iceberg-sink2/python/check.py +++ /dev/null @@ -1,23 +0,0 @@ -from pyspark.sql import SparkSession -import configparser - - -def check_spark_table(args): - spark_config = args['spark'] - spark = SparkSession.builder.remote(spark_config['url']).getOrCreate() - - sqls = [ - "SELECT COUNT(*) FROM s1.t1" - ] - - for sql in sqls: - print(f"Executing sql: {sql}") - result = spark.sql(sql).collect() - print(f"Result is {result}") - - -if __name__ == "__main__": - config = configparser.ConfigParser() - config.read("config.ini") - print({section: dict(config[section]) for section in config.sections()}) - check_spark_table(config) diff --git a/integration_tests/iceberg-sink2/python/config.ini b/integration_tests/iceberg-sink2/python/config.ini deleted file mode 100644 index f8be46e08e3f6..0000000000000 --- a/integration_tests/iceberg-sink2/python/config.ini +++ /dev/null @@ -1,12 +0,0 @@ -[default] -aws_key=a -aws_secret=b - -[spark] -url=sc://localhost:15002 - -[risingwave] -db=dev -user=root -host=127.0.0.1 -port=4566 diff --git a/integration_tests/iceberg-sink2/python/init.py b/integration_tests/iceberg-sink2/python/init.py deleted file mode 100644 index 27ce9b00dcca5..0000000000000 --- a/integration_tests/iceberg-sink2/python/init.py +++ /dev/null @@ -1,92 +0,0 @@ -from pyspark.sql import SparkSession -import configparser -import psycopg2 - - -def init_spark_table(args): - spark_config = args['spark'] - spark = SparkSession.builder.remote(spark_config['url']).getOrCreate() - - init_table_sqls = [ - "CREATE SCHEMA IF NOT EXISTS s1", - "DROP TABLE IF EXISTS s1.t1", - """ - CREATE TABLE s1.t1 - ( - id bigint, - name string, - distance bigint - ) USING iceberg - TBLPROPERTIES ('format-version'='2'); - """, - """INSERT INTO s1.t1 VALUES (1, 'test', 100);""" - ] - - for sql in init_table_sqls: - print(f"Executing sql: {sql}") - spark.sql(sql) - - -def init_risingwave_mv(args): - aws_key = args['default']['aws_key'] - aws_secret = args['default']['aws_secret'] - - rw_config = args['risingwave'] - sqls = [ - "set streaming_parallelism = 4", - """ - CREATE SOURCE bid ( - "id" BIGINT, - "name" VARCHAR, - "distance" BIGINT, - ) with ( - connector = 'datagen', - datagen.split.num = '4', - - - fields.id.kind = 'random', - fields.id.min = '0', - fields.id.max = '1000000000', - fields.id.seed = '100', - - fields.name.kind = 'random', - fields.name.length = '15', - fields.name.seed = '100', - - fields.distance.kind = 'random', - fields.distance.min = '0', - fields.distance.max = '100000', - fields.distance.seed = '100', - - datagen.rows.per.second = '500000' - ) FORMAT PLAIN ENCODE JSON; - """, - f""" - CREATE SINK s1 - AS SELECT * FROM bid - WITH ( - connector='iceberg', - type='append-only', - force_append_only = 'true', - warehouse.path = 's3a://renjie-iceberg-bench/wh', - s3.access.key = '{aws_key}', - s3.secret.key = '{aws_secret}', - s3.region = 'ap-southeast-1', - database.name='s1', - table.name='t1'); - """ - ] - with psycopg2.connect(database=rw_config['db'], user=rw_config['user'], host=rw_config['host'], - port=rw_config['port']) as conn: - with conn.cursor() as cursor: - for sql in sqls: - print(f"Executing sql {sql}") - cursor.execute(sql) - - -if __name__ == "__main__": - config = configparser.ConfigParser() - config.read("config.ini") - print({section: dict(config[section]) for section in config.sections()}) - init_spark_table(config) - init_risingwave_mv(config) diff --git a/integration_tests/iceberg-sink2/python/main.py b/integration_tests/iceberg-sink2/python/main.py new file mode 100644 index 0000000000000..ff0c90d4cf752 --- /dev/null +++ b/integration_tests/iceberg-sink2/python/main.py @@ -0,0 +1,144 @@ +import argparse +import subprocess +from pyspark.sql import SparkSession +import configparser +import psycopg2 +import time + + +def read_config(filename): + config = configparser.ConfigParser() + config.read(filename) + print({section: dict(config[section]) for section in config.sections()}) + return config + + +class DockerCompose(object): + def __init__(self, case_name: str): + self.case_name = case_name + + def case_dir(self): + return f"../docker/{self.case_name}" + + def get_ip(self, container_name): + return subprocess.check_output([ + "docker", "inspect", "-f", "{{range.NetworkSettings.Networks}}{{.IPAddress}}{{end}}", + container_name], cwd=self.case_dir()).decode("utf-8").rstrip() + + def __enter__(self): + subprocess.run(["docker-compose", "up", "-d", "--wait"], cwd=self.case_dir(), check=False) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + subprocess.run(["docker", "compose", "down", "-v", "--remove-orphans"], cwd=self.case_dir(), + capture_output=True, + check=True) + + +def init_spark_table(docker): + spark_ip = docker.get_ip(f"{docker.case_name}-spark-1") + url = f"sc://{spark_ip}:15002" + print(f"Spark url is {url}") + spark = SparkSession.builder.remote(url).getOrCreate() + + init_table_sqls = [ + "CREATE SCHEMA IF NOT EXISTS s1", + "DROP TABLE IF EXISTS s1.t1", + """ + CREATE TABLE s1.t1 + ( + id bigint, + name string, + distance bigint + ) USING iceberg + TBLPROPERTIES ('format-version'='2'); + """, + """INSERT INTO s1.t1 VALUES (1, 'test', 100);""" + ] + + for sql in init_table_sqls: + print(f"Executing sql: {sql}") + spark.sql(sql) + + +def init_risingwave_mv(docker): + config = read_config(f"{docker.case_dir()}/config.ini") + sink_config = config['sink'] + sink_param = ",\n".join([f"{k}='{v}'" for k, v in sink_config.items()]) + sqls = [ + "set streaming_parallelism = 4", + """ + CREATE SOURCE bid ( + "id" BIGINT, + "name" VARCHAR, + "distance" BIGINT, + ) with ( + connector = 'datagen', + datagen.split.num = '4', + + + fields.id.kind = 'random', + fields.id.min = '0', + fields.id.max = '1000000', + fields.id.seed = '100', + + fields.name.kind = 'random', + fields.name.length = '15', + fields.name.seed = '100', + + fields.distance.kind = 'random', + fields.distance.min = '0', + fields.distance.max = '100000', + fields.distance.seed = '100', + + datagen.rows.per.second = '500' + ) FORMAT PLAIN ENCODE JSON; + """, + f""" + CREATE SINK s1 + AS SELECT * FROM bid + WITH ( + {sink_param} + ); + """ + ] + + rw_config = config['risingwave'] + with psycopg2.connect(database=rw_config['db'], user=rw_config['user'], host=rw_config['host'], + port=rw_config['port']) as conn: + with conn.cursor() as cursor: + for sql in sqls: + print(f"Executing sql {sql}") + cursor.execute(sql) + + +def check_spark_table(docker): + spark_ip = docker.get_ip(f"{docker.case_name}-spark-1") + url = f"sc://{spark_ip}:15002" + print(f"Spark url is {url}") + spark = SparkSession.builder.remote(url).getOrCreate() + + sqls = [ + "SELECT COUNT(*) FROM s1.t1" + ] + + for sql in sqls: + print(f"Executing sql: {sql}") + result = spark.sql(sql).collect() + assert result[0][0] > 100, f"Inserted result is too small: {result[0][0]}, test failed" + + +def run_case(case): + with DockerCompose(case) as docker: + init_spark_table(docker) + init_risingwave_mv(docker) + print("Let risingwave to run") + time.sleep(5) + check_spark_table(docker) + + +if __name__ == "__main__": + case_names = ["rest", "storage", "jdbc", "hive"] + for case_name in case_names: + print(f"Running test case: {case_name}") + run_case(case_name) diff --git a/integration_tests/iceberg-sink2/python/poetry.lock b/integration_tests/iceberg-sink2/python/poetry.lock index 32a6a312ef3cc..e87e4aa6fa11a 100644 --- a/integration_tests/iceberg-sink2/python/poetry.lock +++ b/integration_tests/iceberg-sink2/python/poetry.lock @@ -1,14 +1,14 @@ -# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.7.1 and should not be changed by hand. [[package]] name = "googleapis-common-protos" -version = "1.60.0" +version = "1.62.0" description = "Common protobufs used in Google APIs" optional = false python-versions = ">=3.7" files = [ - {file = "googleapis-common-protos-1.60.0.tar.gz", hash = "sha256:e73ebb404098db405ba95d1e1ae0aa91c3e15a71da031a2eeb6b2e23e7bc3708"}, - {file = "googleapis_common_protos-1.60.0-py2.py3-none-any.whl", hash = "sha256:69f9bbcc6acde92cab2db95ce30a70bd2b81d20b12eff3f1aabaffcbe8a93918"}, + {file = "googleapis-common-protos-1.62.0.tar.gz", hash = "sha256:83f0ece9f94e5672cced82f592d2a5edf527a96ed1794f0bab36d5735c996277"}, + {file = "googleapis_common_protos-1.62.0-py2.py3-none-any.whl", hash = "sha256:4750113612205514f9f6aa4cb00d523a94f3e8c06c5ad2fee466387dc4875f07"}, ] [package.dependencies] @@ -19,109 +19,121 @@ grpc = ["grpcio (>=1.44.0,<2.0.0.dev0)"] [[package]] name = "grpcio" -version = "1.56.2" +version = "1.62.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.7" files = [ - {file = "grpcio-1.56.2-cp310-cp310-linux_armv7l.whl", hash = "sha256:bf0b9959e673505ee5869950642428046edb91f99942607c2ecf635f8a4b31c9"}, - {file = "grpcio-1.56.2-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:5144feb20fe76e73e60c7d73ec3bf54f320247d1ebe737d10672480371878b48"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:a72797549935c9e0b9bc1def1768c8b5a709538fa6ab0678e671aec47ebfd55e"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c3f3237a57e42f79f1e560726576aedb3a7ef931f4e3accb84ebf6acc485d316"}, - {file = "grpcio-1.56.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:900bc0096c2ca2d53f2e5cebf98293a7c32f532c4aeb926345e9747452233950"}, - {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:97e0efaebbfd222bcaac2f1735c010c1d3b167112d9d237daebbeedaaccf3d1d"}, - {file = "grpcio-1.56.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c0c85c5cbe8b30a32fa6d802588d55ffabf720e985abe9590c7c886919d875d4"}, - {file = "grpcio-1.56.2-cp310-cp310-win32.whl", hash = "sha256:06e84ad9ae7668a109e970c7411e7992751a116494cba7c4fb877656527f9a57"}, - {file = "grpcio-1.56.2-cp310-cp310-win_amd64.whl", hash = "sha256:10954662f77dc36c9a1fb5cc4a537f746580d6b5734803be1e587252682cda8d"}, - {file = "grpcio-1.56.2-cp311-cp311-linux_armv7l.whl", hash = "sha256:c435f5ce1705de48e08fcbcfaf8aee660d199c90536e3e06f2016af7d6a938dd"}, - {file = "grpcio-1.56.2-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:6108e5933eb8c22cd3646e72d5b54772c29f57482fd4c41a0640aab99eb5071d"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:8391cea5ce72f4a12368afd17799474015d5d3dc00c936a907eb7c7eaaea98a5"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:750de923b456ca8c0f1354d6befca45d1f3b3a789e76efc16741bd4132752d95"}, - {file = "grpcio-1.56.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fda2783c12f553cdca11c08e5af6eecbd717280dc8fbe28a110897af1c15a88c"}, - {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:9e04d4e4cfafa7c5264e535b5d28e786f0571bea609c3f0aaab13e891e933e9c"}, - {file = "grpcio-1.56.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:89a49cc5ad08a38b6141af17e00d1dd482dc927c7605bc77af457b5a0fca807c"}, - {file = "grpcio-1.56.2-cp311-cp311-win32.whl", hash = "sha256:6a007a541dff984264981fbafeb052bfe361db63578948d857907df9488d8774"}, - {file = "grpcio-1.56.2-cp311-cp311-win_amd64.whl", hash = "sha256:af4063ef2b11b96d949dccbc5a987272f38d55c23c4c01841ea65a517906397f"}, - {file = "grpcio-1.56.2-cp37-cp37m-linux_armv7l.whl", hash = "sha256:a6ff459dac39541e6a2763a4439c4ca6bc9ecb4acc05a99b79246751f9894756"}, - {file = "grpcio-1.56.2-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:f20fd21f7538f8107451156dd1fe203300b79a9ddceba1ee0ac8132521a008ed"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:d1fbad1f9077372b6587ec589c1fc120b417b6c8ad72d3e3cc86bbbd0a3cee93"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ee26e9dfb3996aff7c870f09dc7ad44a5f6732b8bdb5a5f9905737ac6fd4ef1"}, - {file = "grpcio-1.56.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4c60abd950d6de3e4f1ddbc318075654d275c29c846ab6a043d6ed2c52e4c8c"}, - {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1c31e52a04e62c8577a7bf772b3e7bed4df9c9e0dd90f92b6ffa07c16cab63c9"}, - {file = "grpcio-1.56.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:345356b307cce5d14355e8e055b4ca5f99bc857c33a3dc1ddbc544fca9cd0475"}, - {file = "grpcio-1.56.2-cp37-cp37m-win_amd64.whl", hash = "sha256:42e63904ee37ae46aa23de50dac8b145b3596f43598fa33fe1098ab2cbda6ff5"}, - {file = "grpcio-1.56.2-cp38-cp38-linux_armv7l.whl", hash = "sha256:7c5ede2e2558f088c49a1ddda19080e4c23fb5d171de80a726b61b567e3766ed"}, - {file = "grpcio-1.56.2-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:33971197c47965cc1d97d78d842163c283e998223b151bab0499b951fd2c0b12"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:d39f5d4af48c138cb146763eda14eb7d8b3ccbbec9fe86fb724cd16e0e914c64"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ded637176addc1d3eef35331c39acc598bac550d213f0a1bedabfceaa2244c87"}, - {file = "grpcio-1.56.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c90da4b124647547a68cf2f197174ada30c7bb9523cb976665dfd26a9963d328"}, - {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:3ccb621749a81dc7755243665a70ce45536ec413ef5818e013fe8dfbf5aa497b"}, - {file = "grpcio-1.56.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:4eb37dd8dd1aa40d601212afa27ca5be255ba792e2e0b24d67b8af5e012cdb7d"}, - {file = "grpcio-1.56.2-cp38-cp38-win32.whl", hash = "sha256:ddb4a6061933bd9332b74eac0da25f17f32afa7145a33a0f9711ad74f924b1b8"}, - {file = "grpcio-1.56.2-cp38-cp38-win_amd64.whl", hash = "sha256:8940d6de7068af018dfa9a959a3510e9b7b543f4c405e88463a1cbaa3b2b379a"}, - {file = "grpcio-1.56.2-cp39-cp39-linux_armv7l.whl", hash = "sha256:51173e8fa6d9a2d85c14426bdee5f5c4a0654fd5fddcc21fe9d09ab0f6eb8b35"}, - {file = "grpcio-1.56.2-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:373b48f210f43327a41e397391715cd11cfce9ded2fe76a5068f9bacf91cc226"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:42a3bbb2bc07aef72a7d97e71aabecaf3e4eb616d39e5211e2cfe3689de860ca"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5344be476ac37eb9c9ad09c22f4ea193c1316bf074f1daf85bddb1b31fda5116"}, - {file = "grpcio-1.56.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c3fa3ab0fb200a2c66493828ed06ccd1a94b12eddbfb985e7fd3e5723ff156c6"}, - {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:b975b85d1d5efc36cf8b237c5f3849b64d1ba33d6282f5e991f28751317504a1"}, - {file = "grpcio-1.56.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:cbdf2c498e077282cd427cfd88bdce4668019791deef0be8155385ab2ba7837f"}, - {file = "grpcio-1.56.2-cp39-cp39-win32.whl", hash = "sha256:139f66656a762572ae718fa0d1f2dce47c05e9fbf7a16acd704c354405b97df9"}, - {file = "grpcio-1.56.2-cp39-cp39-win_amd64.whl", hash = "sha256:830215173ad45d670140ff99aac3b461f9be9a6b11bee1a17265aaaa746a641a"}, - {file = "grpcio-1.56.2.tar.gz", hash = "sha256:0ff789ae7d8ddd76d2ac02e7d13bfef6fc4928ac01e1dcaa182be51b6bcc0aaa"}, + {file = "grpcio-1.62.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:136ffd79791b1eddda8d827b607a6285474ff8a1a5735c4947b58c481e5e4271"}, + {file = "grpcio-1.62.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:d6a56ba703be6b6267bf19423d888600c3f574ac7c2cc5e6220af90662a4d6b0"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:4cd356211579043fce9f52acc861e519316fff93980a212c8109cca8f47366b6"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e803e9b58d8f9b4ff0ea991611a8d51b31c68d2e24572cd1fe85e99e8cc1b4f8"}, + {file = "grpcio-1.62.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f4c04fe33039b35b97c02d2901a164bbbb2f21fb9c4e2a45a959f0b044c3512c"}, + {file = "grpcio-1.62.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:95370c71b8c9062f9ea033a0867c4c73d6f0ff35113ebd2618171ec1f1e903e0"}, + {file = "grpcio-1.62.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c912688acc05e4ff012c8891803659d6a8a8b5106f0f66e0aed3fb7e77898fa6"}, + {file = "grpcio-1.62.0-cp310-cp310-win32.whl", hash = "sha256:821a44bd63d0f04e33cf4ddf33c14cae176346486b0df08b41a6132b976de5fc"}, + {file = "grpcio-1.62.0-cp310-cp310-win_amd64.whl", hash = "sha256:81531632f93fece32b2762247c4c169021177e58e725494f9a746ca62c83acaa"}, + {file = "grpcio-1.62.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:3fa15850a6aba230eed06b236287c50d65a98f05054a0f01ccedf8e1cc89d57f"}, + {file = "grpcio-1.62.0-cp311-cp311-macosx_10_10_universal2.whl", hash = "sha256:36df33080cd7897623feff57831eb83c98b84640b016ce443305977fac7566fb"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:7a195531828b46ea9c4623c47e1dc45650fc7206f8a71825898dd4c9004b0928"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ab140a3542bbcea37162bdfc12ce0d47a3cda3f2d91b752a124cc9fe6776a9e2"}, + {file = "grpcio-1.62.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7f9d6c3223914abb51ac564dc9c3782d23ca445d2864321b9059d62d47144021"}, + {file = "grpcio-1.62.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:fbe0c20ce9a1cff75cfb828b21f08d0a1ca527b67f2443174af6626798a754a4"}, + {file = "grpcio-1.62.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:38f69de9c28c1e7a8fd24e4af4264726637b72f27c2099eaea6e513e7142b47e"}, + {file = "grpcio-1.62.0-cp311-cp311-win32.whl", hash = "sha256:ce1aafdf8d3f58cb67664f42a617af0e34555fe955450d42c19e4a6ad41c84bd"}, + {file = "grpcio-1.62.0-cp311-cp311-win_amd64.whl", hash = "sha256:eef1d16ac26c5325e7d39f5452ea98d6988c700c427c52cbc7ce3201e6d93334"}, + {file = "grpcio-1.62.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8aab8f90b2a41208c0a071ec39a6e5dbba16fd827455aaa070fec241624ccef8"}, + {file = "grpcio-1.62.0-cp312-cp312-macosx_10_10_universal2.whl", hash = "sha256:62aa1659d8b6aad7329ede5d5b077e3d71bf488d85795db517118c390358d5f6"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:0d7ae7fc7dbbf2d78d6323641ded767d9ec6d121aaf931ec4a5c50797b886532"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f359d635ee9428f0294bea062bb60c478a8ddc44b0b6f8e1f42997e5dc12e2ee"}, + {file = "grpcio-1.62.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d48e5b1f8f4204889f1acf30bb57c30378e17c8d20df5acbe8029e985f735c"}, + {file = "grpcio-1.62.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:662d3df5314ecde3184cf87ddd2c3a66095b3acbb2d57a8cada571747af03873"}, + {file = "grpcio-1.62.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:92cdb616be44c8ac23a57cce0243af0137a10aa82234f23cd46e69e115071388"}, + {file = "grpcio-1.62.0-cp312-cp312-win32.whl", hash = "sha256:0b9179478b09ee22f4a36b40ca87ad43376acdccc816ce7c2193a9061bf35701"}, + {file = "grpcio-1.62.0-cp312-cp312-win_amd64.whl", hash = "sha256:614c3ed234208e76991992342bab725f379cc81c7dd5035ee1de2f7e3f7a9842"}, + {file = "grpcio-1.62.0-cp37-cp37m-linux_armv7l.whl", hash = "sha256:7e1f51e2a460b7394670fdb615e26d31d3260015154ea4f1501a45047abe06c9"}, + {file = "grpcio-1.62.0-cp37-cp37m-macosx_10_10_universal2.whl", hash = "sha256:bcff647e7fe25495e7719f779cc219bbb90b9e79fbd1ce5bda6aae2567f469f2"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_aarch64.whl", hash = "sha256:56ca7ba0b51ed0de1646f1735154143dcbdf9ec2dbe8cc6645def299bb527ca1"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:2e84bfb2a734e4a234b116be208d6f0214e68dcf7804306f97962f93c22a1839"}, + {file = "grpcio-1.62.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c1488b31a521fbba50ae86423f5306668d6f3a46d124f7819c603979fc538c4"}, + {file = "grpcio-1.62.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:98d8f4eb91f1ce0735bf0b67c3b2a4fea68b52b2fd13dc4318583181f9219b4b"}, + {file = "grpcio-1.62.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:b3d3d755cfa331d6090e13aac276d4a3fb828bf935449dc16c3d554bf366136b"}, + {file = "grpcio-1.62.0-cp37-cp37m-win_amd64.whl", hash = "sha256:a33f2bfd8a58a02aab93f94f6c61279be0f48f99fcca20ebaee67576cd57307b"}, + {file = "grpcio-1.62.0-cp38-cp38-linux_armv7l.whl", hash = "sha256:5e709f7c8028ce0443bddc290fb9c967c1e0e9159ef7a030e8c21cac1feabd35"}, + {file = "grpcio-1.62.0-cp38-cp38-macosx_10_10_universal2.whl", hash = "sha256:2f3d9a4d0abb57e5f49ed5039d3ed375826c2635751ab89dcc25932ff683bbb6"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_aarch64.whl", hash = "sha256:62ccb92f594d3d9fcd00064b149a0187c246b11e46ff1b7935191f169227f04c"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:921148f57c2e4b076af59a815467d399b7447f6e0ee10ef6d2601eb1e9c7f402"}, + {file = "grpcio-1.62.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f897b16190b46bc4d4aaf0a32a4b819d559a37a756d7c6b571e9562c360eed72"}, + {file = "grpcio-1.62.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:1bc8449084fe395575ed24809752e1dc4592bb70900a03ca42bf236ed5bf008f"}, + {file = "grpcio-1.62.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:81d444e5e182be4c7856cd33a610154fe9ea1726bd071d07e7ba13fafd202e38"}, + {file = "grpcio-1.62.0-cp38-cp38-win32.whl", hash = "sha256:88f41f33da3840b4a9bbec68079096d4caf629e2c6ed3a72112159d570d98ebe"}, + {file = "grpcio-1.62.0-cp38-cp38-win_amd64.whl", hash = "sha256:fc2836cb829895ee190813446dce63df67e6ed7b9bf76060262c55fcd097d270"}, + {file = "grpcio-1.62.0-cp39-cp39-linux_armv7l.whl", hash = "sha256:fcc98cff4084467839d0a20d16abc2a76005f3d1b38062464d088c07f500d170"}, + {file = "grpcio-1.62.0-cp39-cp39-macosx_10_10_universal2.whl", hash = "sha256:0d3dee701e48ee76b7d6fbbba18ba8bc142e5b231ef7d3d97065204702224e0e"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_aarch64.whl", hash = "sha256:b7a6be562dd18e5d5bec146ae9537f20ae1253beb971c0164f1e8a2f5a27e829"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:29cb592c4ce64a023712875368bcae13938c7f03e99f080407e20ffe0a9aa33b"}, + {file = "grpcio-1.62.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1eda79574aec8ec4d00768dcb07daba60ed08ef32583b62b90bbf274b3c279f7"}, + {file = "grpcio-1.62.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:7eea57444a354ee217fda23f4b479a4cdfea35fb918ca0d8a0e73c271e52c09c"}, + {file = "grpcio-1.62.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:0e97f37a3b7c89f9125b92d22e9c8323f4e76e7993ba7049b9f4ccbe8bae958a"}, + {file = "grpcio-1.62.0-cp39-cp39-win32.whl", hash = "sha256:39cd45bd82a2e510e591ca2ddbe22352e8413378852ae814549c162cf3992a93"}, + {file = "grpcio-1.62.0-cp39-cp39-win_amd64.whl", hash = "sha256:b71c65427bf0ec6a8b48c68c17356cb9fbfc96b1130d20a07cb462f4e4dcdcd5"}, + {file = "grpcio-1.62.0.tar.gz", hash = "sha256:748496af9238ac78dcd98cce65421f1adce28c3979393e3609683fcd7f3880d7"}, ] [package.extras] -protobuf = ["grpcio-tools (>=1.56.2)"] +protobuf = ["grpcio-tools (>=1.62.0)"] [[package]] name = "grpcio-status" -version = "1.56.2" +version = "1.62.0" description = "Status proto mapping for gRPC" optional = false python-versions = ">=3.6" files = [ - {file = "grpcio-status-1.56.2.tar.gz", hash = "sha256:a046b2c0118df4a5687f4585cca9d3c3bae5c498c4dff055dcb43fb06a1180c8"}, - {file = "grpcio_status-1.56.2-py3-none-any.whl", hash = "sha256:63f3842867735f59f5d70e723abffd2e8501a6bcd915612a1119e52f10614782"}, + {file = "grpcio-status-1.62.0.tar.gz", hash = "sha256:0d693e9c09880daeaac060d0c3dba1ae470a43c99e5d20dfeafd62cf7e08a85d"}, + {file = "grpcio_status-1.62.0-py3-none-any.whl", hash = "sha256:3baac03fcd737310e67758c4082a188107f771d32855bce203331cd4c9aa687a"}, ] [package.dependencies] googleapis-common-protos = ">=1.5.5" -grpcio = ">=1.56.2" +grpcio = ">=1.62.0" protobuf = ">=4.21.6" [[package]] name = "numpy" -version = "1.25.2" +version = "1.24.4" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.9" +python-versions = ">=3.8" files = [ - {file = "numpy-1.25.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:db3ccc4e37a6873045580d413fe79b68e47a681af8db2e046f1dacfa11f86eb3"}, - {file = "numpy-1.25.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:90319e4f002795ccfc9050110bbbaa16c944b1c37c0baeea43c5fb881693ae1f"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:dfe4a913e29b418d096e696ddd422d8a5d13ffba4ea91f9f60440a3b759b0187"}, - {file = "numpy-1.25.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f08f2e037bba04e707eebf4bc934f1972a315c883a9e0ebfa8a7756eabf9e357"}, - {file = "numpy-1.25.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:bec1e7213c7cb00d67093247f8c4db156fd03075f49876957dca4711306d39c9"}, - {file = "numpy-1.25.2-cp310-cp310-win32.whl", hash = "sha256:7dc869c0c75988e1c693d0e2d5b26034644399dd929bc049db55395b1379e044"}, - {file = "numpy-1.25.2-cp310-cp310-win_amd64.whl", hash = "sha256:834b386f2b8210dca38c71a6e0f4fd6922f7d3fcff935dbe3a570945acb1b545"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c5462d19336db4560041517dbb7759c21d181a67cb01b36ca109b2ae37d32418"}, - {file = "numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c5652ea24d33585ea39eb6a6a15dac87a1206a692719ff45d53c5282e66d4a8f"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d60fbae8e0019865fc4784745814cff1c421df5afee233db6d88ab4f14655a2"}, - {file = "numpy-1.25.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:60e7f0f7f6d0eee8364b9a6304c2845b9c491ac706048c7e8cf47b83123b8dbf"}, - {file = "numpy-1.25.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:bb33d5a1cf360304754913a350edda36d5b8c5331a8237268c48f91253c3a364"}, - {file = "numpy-1.25.2-cp311-cp311-win32.whl", hash = "sha256:5883c06bb92f2e6c8181df7b39971a5fb436288db58b5a1c3967702d4278691d"}, - {file = "numpy-1.25.2-cp311-cp311-win_amd64.whl", hash = "sha256:5c97325a0ba6f9d041feb9390924614b60b99209a71a69c876f71052521d42a4"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:b79e513d7aac42ae918db3ad1341a015488530d0bb2a6abcbdd10a3a829ccfd3"}, - {file = "numpy-1.25.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:eb942bfb6f84df5ce05dbf4b46673ffed0d3da59f13635ea9b926af3deb76926"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e0746410e73384e70d286f93abf2520035250aad8c5714240b0492a7302fdca"}, - {file = "numpy-1.25.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d7806500e4f5bdd04095e849265e55de20d8cc4b661b038957354327f6d9b295"}, - {file = "numpy-1.25.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8b77775f4b7df768967a7c8b3567e309f617dd5e99aeb886fa14dc1a0791141f"}, - {file = "numpy-1.25.2-cp39-cp39-win32.whl", hash = "sha256:2792d23d62ec51e50ce4d4b7d73de8f67a2fd3ea710dcbc8563a51a03fb07b01"}, - {file = "numpy-1.25.2-cp39-cp39-win_amd64.whl", hash = "sha256:76b4115d42a7dfc5d485d358728cdd8719be33cc5ec6ec08632a5d6fca2ed380"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:1a1329e26f46230bf77b02cc19e900db9b52f398d6722ca853349a782d4cff55"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4c3abc71e8b6edba80a01a52e66d83c5d14433cbcd26a40c329ec7ed09f37901"}, - {file = "numpy-1.25.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:1b9735c27cea5d995496f46a8b1cd7b408b3f34b6d50459d9ac8fe3a20cc17bf"}, - {file = "numpy-1.25.2.tar.gz", hash = "sha256:fd608e19c8d7c55021dffd43bfe5492fab8cc105cc8986f813f8c3c048b38760"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, + {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, + {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, + {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, + {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, + {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, + {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, + {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, + {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, + {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, ] [[package]] @@ -159,7 +171,11 @@ files = [ ] [package.dependencies] -numpy = {version = ">=1.23.2", markers = "python_version >= \"3.11\""} +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.1" @@ -189,24 +205,22 @@ xml = ["lxml (>=4.6.3)"] [[package]] name = "protobuf" -version = "4.24.0" +version = "4.25.3" description = "" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" files = [ - {file = "protobuf-4.24.0-cp310-abi3-win32.whl", hash = "sha256:81cb9c4621d2abfe181154354f63af1c41b00a4882fb230b4425cbaed65e8f52"}, - {file = "protobuf-4.24.0-cp310-abi3-win_amd64.whl", hash = "sha256:6c817cf4a26334625a1904b38523d1b343ff8b637d75d2c8790189a4064e51c3"}, - {file = "protobuf-4.24.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:ae97b5de10f25b7a443b40427033e545a32b0e9dda17bcd8330d70033379b3e5"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:567fe6b0647494845d0849e3d5b260bfdd75692bf452cdc9cb660d12457c055d"}, - {file = "protobuf-4.24.0-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:a6b1ca92ccabfd9903c0c7dde8876221dc7d8d87ad5c42e095cc11b15d3569c7"}, - {file = "protobuf-4.24.0-cp37-cp37m-win32.whl", hash = "sha256:a38400a692fd0c6944c3c58837d112f135eb1ed6cdad5ca6c5763336e74f1a04"}, - {file = "protobuf-4.24.0-cp37-cp37m-win_amd64.whl", hash = "sha256:5ab19ee50037d4b663c02218a811a5e1e7bb30940c79aac385b96e7a4f9daa61"}, - {file = "protobuf-4.24.0-cp38-cp38-win32.whl", hash = "sha256:e8834ef0b4c88666ebb7c7ec18045aa0f4325481d724daa624a4cf9f28134653"}, - {file = "protobuf-4.24.0-cp38-cp38-win_amd64.whl", hash = "sha256:8bb52a2be32db82ddc623aefcedfe1e0eb51da60e18fcc908fb8885c81d72109"}, - {file = "protobuf-4.24.0-cp39-cp39-win32.whl", hash = "sha256:ae7a1835721086013de193311df858bc12cd247abe4ef9710b715d930b95b33e"}, - {file = "protobuf-4.24.0-cp39-cp39-win_amd64.whl", hash = "sha256:44825e963008f8ea0d26c51911c30d3e82e122997c3c4568fd0385dd7bacaedf"}, - {file = "protobuf-4.24.0-py3-none-any.whl", hash = "sha256:82e6e9ebdd15b8200e8423676eab38b774624d6a1ad696a60d86a2ac93f18201"}, - {file = "protobuf-4.24.0.tar.gz", hash = "sha256:5d0ceb9de6e08311832169e601d1fc71bd8e8c779f3ee38a97a78554945ecb85"}, + {file = "protobuf-4.25.3-cp310-abi3-win32.whl", hash = "sha256:d4198877797a83cbfe9bffa3803602bbe1625dc30d8a097365dbc762e5790faa"}, + {file = "protobuf-4.25.3-cp310-abi3-win_amd64.whl", hash = "sha256:209ba4cc916bab46f64e56b85b090607a676f66b473e6b762e6f1d9d591eb2e8"}, + {file = "protobuf-4.25.3-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:f1279ab38ecbfae7e456a108c5c0681e4956d5b1090027c1de0f934dfdb4b35c"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_aarch64.whl", hash = "sha256:e7cb0ae90dd83727f0c0718634ed56837bfeeee29a5f82a7514c03ee1364c019"}, + {file = "protobuf-4.25.3-cp37-abi3-manylinux2014_x86_64.whl", hash = "sha256:7c8daa26095f82482307bc717364e7c13f4f1c99659be82890dcfc215194554d"}, + {file = "protobuf-4.25.3-cp38-cp38-win32.whl", hash = "sha256:f4f118245c4a087776e0a8408be33cf09f6c547442c00395fbfb116fac2f8ac2"}, + {file = "protobuf-4.25.3-cp38-cp38-win_amd64.whl", hash = "sha256:c053062984e61144385022e53678fbded7aea14ebb3e0305ae3592fb219ccfa4"}, + {file = "protobuf-4.25.3-cp39-cp39-win32.whl", hash = "sha256:19b270aeaa0099f16d3ca02628546b8baefe2955bbe23224aaf856134eccf1e4"}, + {file = "protobuf-4.25.3-cp39-cp39-win_amd64.whl", hash = "sha256:e3c97a1555fd6388f857770ff8b9703083de6bf1f9274a002a332d65fbb56c8c"}, + {file = "protobuf-4.25.3-py3-none-any.whl", hash = "sha256:f0700d54bcf45424477e46a9f0944155b46fb0639d69728739c0e47bab83f2b9"}, + {file = "protobuf-4.25.3.tar.gz", hash = "sha256:25b5d0b42fd000320bd7830b349e3b696435f3b329810427a6bcce6a5492cc5c"}, ] [[package]] @@ -303,51 +317,51 @@ files = [ [[package]] name = "pyarrow" -version = "14.0.1" +version = "15.0.0" description = "Python library for Apache Arrow" optional = false python-versions = ">=3.8" files = [ - {file = "pyarrow-14.0.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:96d64e5ba7dceb519a955e5eeb5c9adcfd63f73a56aea4722e2cc81364fc567a"}, - {file = "pyarrow-14.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1a8ae88c0038d1bc362a682320112ee6774f006134cd5afc291591ee4bc06505"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0f6f053cb66dc24091f5511e5920e45c83107f954a21032feadc7b9e3a8e7851"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:906b0dc25f2be12e95975722f1e60e162437023f490dbd80d0deb7375baf3171"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:78d4a77a46a7de9388b653af1c4ce539350726cd9af62e0831e4f2bd0c95a2f4"}, - {file = "pyarrow-14.0.1-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:06ca79080ef89d6529bb8e5074d4b4f6086143b2520494fcb7cf8a99079cde93"}, - {file = "pyarrow-14.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:32542164d905002c42dff896efdac79b3bdd7291b1b74aa292fac8450d0e4dcd"}, - {file = "pyarrow-14.0.1-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:c7331b4ed3401b7ee56f22c980608cf273f0380f77d0f73dd3c185f78f5a6220"}, - {file = "pyarrow-14.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:922e8b49b88da8633d6cac0e1b5a690311b6758d6f5d7c2be71acb0f1e14cd61"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58c889851ca33f992ea916b48b8540735055201b177cb0dcf0596a495a667b00"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:30d8494870d9916bb53b2a4384948491444741cb9a38253c590e21f836b01222"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:be28e1a07f20391bb0b15ea03dcac3aade29fc773c5eb4bee2838e9b2cdde0cb"}, - {file = "pyarrow-14.0.1-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:981670b4ce0110d8dcb3246410a4aabf5714db5d8ea63b15686bce1c914b1f83"}, - {file = "pyarrow-14.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:4756a2b373a28f6166c42711240643fb8bd6322467e9aacabd26b488fa41ec23"}, - {file = "pyarrow-14.0.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:cf87e2cec65dd5cf1aa4aba918d523ef56ef95597b545bbaad01e6433851aa10"}, - {file = "pyarrow-14.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:470ae0194fbfdfbf4a6b65b4f9e0f6e1fa0ea5b90c1ee6b65b38aecee53508c8"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6263cffd0c3721c1e348062997babdf0151301f7353010c9c9a8ed47448f82ab"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a8089d7e77d1455d529dbd7cff08898bbb2666ee48bc4085203af1d826a33cc"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:fada8396bc739d958d0b81d291cfd201126ed5e7913cb73de6bc606befc30226"}, - {file = "pyarrow-14.0.1-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:2a145dab9ed7849fc1101bf03bcdc69913547f10513fdf70fc3ab6c0a50c7eee"}, - {file = "pyarrow-14.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:05fe7994745b634c5fb16ce5717e39a1ac1fac3e2b0795232841660aa76647cd"}, - {file = "pyarrow-14.0.1-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:a8eeef015ae69d104c4c3117a6011e7e3ecd1abec79dc87fd2fac6e442f666ee"}, - {file = "pyarrow-14.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:3c76807540989fe8fcd02285dd15e4f2a3da0b09d27781abec3adc265ddbeba1"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:450e4605e3c20e558485f9161a79280a61c55efe585d51513c014de9ae8d393f"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:323cbe60210173ffd7db78bfd50b80bdd792c4c9daca8843ef3cd70b186649db"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:0140c7e2b740e08c5a459439d87acd26b747fc408bde0a8806096ee0baaa0c15"}, - {file = "pyarrow-14.0.1-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:e592e482edd9f1ab32f18cd6a716c45b2c0f2403dc2af782f4e9674952e6dd27"}, - {file = "pyarrow-14.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:d264ad13605b61959f2ae7c1d25b1a5b8505b112715c961418c8396433f213ad"}, - {file = "pyarrow-14.0.1-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:01e44de9749cddc486169cb632f3c99962318e9dacac7778315a110f4bf8a450"}, - {file = "pyarrow-14.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d0351fecf0e26e152542bc164c22ea2a8e8c682726fce160ce4d459ea802d69c"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33c1f6110c386464fd2e5e4ea3624466055bbe681ff185fd6c9daa98f30a3f9a"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:11e045dfa09855b6d3e7705a37c42e2dc2c71d608fab34d3c23df2e02df9aec3"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:097828b55321897db0e1dbfc606e3ff8101ae5725673498cbfa7754ee0da80e4"}, - {file = "pyarrow-14.0.1-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:1daab52050a1c48506c029e6fa0944a7b2436334d7e44221c16f6f1b2cc9c510"}, - {file = "pyarrow-14.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:3f6d5faf4f1b0d5a7f97be987cf9e9f8cd39902611e818fe134588ee99bf0283"}, - {file = "pyarrow-14.0.1.tar.gz", hash = "sha256:b8b3f4fe8d4ec15e1ef9b599b94683c5216adaed78d5cb4c606180546d1e2ee1"}, + {file = "pyarrow-15.0.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:0a524532fd6dd482edaa563b686d754c70417c2f72742a8c990b322d4c03a15d"}, + {file = "pyarrow-15.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:60a6bdb314affa9c2e0d5dddf3d9cbb9ef4a8dddaa68669975287d47ece67642"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:66958fd1771a4d4b754cd385835e66a3ef6b12611e001d4e5edfcef5f30391e2"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f500956a49aadd907eaa21d4fff75f73954605eaa41f61cb94fb008cf2e00c6"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:6f87d9c4f09e049c2cade559643424da84c43a35068f2a1c4653dc5b1408a929"}, + {file = "pyarrow-15.0.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:85239b9f93278e130d86c0e6bb455dcb66fc3fd891398b9d45ace8799a871a1e"}, + {file = "pyarrow-15.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:5b8d43e31ca16aa6e12402fcb1e14352d0d809de70edd185c7650fe80e0769e3"}, + {file = "pyarrow-15.0.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:fa7cd198280dbd0c988df525e50e35b5d16873e2cdae2aaaa6363cdb64e3eec5"}, + {file = "pyarrow-15.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8780b1a29d3c8b21ba6b191305a2a607de2e30dab399776ff0aa09131e266340"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe0ec198ccc680f6c92723fadcb97b74f07c45ff3fdec9dd765deb04955ccf19"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:036a7209c235588c2f07477fe75c07e6caced9b7b61bb897c8d4e52c4b5f9555"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:2bd8a0e5296797faf9a3294e9fa2dc67aa7f10ae2207920dbebb785c77e9dbe5"}, + {file = "pyarrow-15.0.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:e8ebed6053dbe76883a822d4e8da36860f479d55a762bd9e70d8494aed87113e"}, + {file = "pyarrow-15.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:17d53a9d1b2b5bd7d5e4cd84d018e2a45bc9baaa68f7e6e3ebed45649900ba99"}, + {file = "pyarrow-15.0.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:9950a9c9df24090d3d558b43b97753b8f5867fb8e521f29876aa021c52fda351"}, + {file = "pyarrow-15.0.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:003d680b5e422d0204e7287bb3fa775b332b3fce2996aa69e9adea23f5c8f970"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f75fce89dad10c95f4bf590b765e3ae98bcc5ba9f6ce75adb828a334e26a3d40"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca9cb0039923bec49b4fe23803807e4ef39576a2bec59c32b11296464623dc2"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:9ed5a78ed29d171d0acc26a305a4b7f83c122d54ff5270810ac23c75813585e4"}, + {file = "pyarrow-15.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6eda9e117f0402dfcd3cd6ec9bfee89ac5071c48fc83a84f3075b60efa96747f"}, + {file = "pyarrow-15.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:9a3a6180c0e8f2727e6f1b1c87c72d3254cac909e609f35f22532e4115461177"}, + {file = "pyarrow-15.0.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:19a8918045993349b207de72d4576af0191beef03ea655d8bdb13762f0cd6eac"}, + {file = "pyarrow-15.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:d0ec076b32bacb6666e8813a22e6e5a7ef1314c8069d4ff345efa6246bc38593"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5db1769e5d0a77eb92344c7382d6543bea1164cca3704f84aa44e26c67e320fb"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e2617e3bf9df2a00020dd1c1c6dce5cc343d979efe10bc401c0632b0eef6ef5b"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:d31c1d45060180131caf10f0f698e3a782db333a422038bf7fe01dace18b3a31"}, + {file = "pyarrow-15.0.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:c8c287d1d479de8269398b34282e206844abb3208224dbdd7166d580804674b7"}, + {file = "pyarrow-15.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:07eb7f07dc9ecbb8dace0f58f009d3a29ee58682fcdc91337dfeb51ea618a75b"}, + {file = "pyarrow-15.0.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:47af7036f64fce990bb8a5948c04722e4e3ea3e13b1007ef52dfe0aa8f23cf7f"}, + {file = "pyarrow-15.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:93768ccfff85cf044c418bfeeafce9a8bb0cee091bd8fd19011aff91e58de540"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f6ee87fd6892700960d90abb7b17a72a5abb3b64ee0fe8db6c782bcc2d0dc0b4"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:001fca027738c5f6be0b7a3159cc7ba16a5c52486db18160909a0831b063c4e4"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:d1c48648f64aec09accf44140dccb92f4f94394b8d79976c426a5b79b11d4fa7"}, + {file = "pyarrow-15.0.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:972a0141be402bb18e3201448c8ae62958c9c7923dfaa3b3d4530c835ac81aed"}, + {file = "pyarrow-15.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:f01fc5cf49081426429127aa2d427d9d98e1cb94a32cb961d583a70b7c4504e6"}, + {file = "pyarrow-15.0.0.tar.gz", hash = "sha256:876858f549d540898f927eba4ef77cd549ad8d24baa3207cf1b72e5788b50e83"}, ] [package.dependencies] -numpy = ">=1.16.6" +numpy = ">=1.16.6,<2" [[package]] name = "pyspark" @@ -377,13 +391,13 @@ sql = ["numpy (>=1.15)", "pandas (>=1.0.5)", "pyarrow (>=1.0.0)"] [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, ] [package.dependencies] @@ -391,13 +405,13 @@ six = ">=1.5" [[package]] name = "pytz" -version = "2023.3" +version = "2024.1" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" files = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, ] [[package]] @@ -413,16 +427,16 @@ files = [ [[package]] name = "tzdata" -version = "2023.3" +version = "2024.1" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" files = [ - {file = "tzdata-2023.3-py2.py3-none-any.whl", hash = "sha256:7e65763eef3120314099b6939b5546db7adce1e7d6f2e179e3df563c70511eda"}, - {file = "tzdata-2023.3.tar.gz", hash = "sha256:11ef1e08e54acb0d4f95bdb1be05da659673de4acbd21bf9c69e94cc5e907a3a"}, + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, ] [metadata] lock-version = "2.0" -python-versions = "^3.11" -content-hash = "42b788fcc212aceceeff3c9e2743d39c048a323068a3b49940b7efa830e6863e" +python-versions = "^3.8" +content-hash = "8278b86130f9910c9769a1e666992ba9e115cb50e829b0c9cf4c4aee1b2efbf8" diff --git a/integration_tests/iceberg-sink2/python/pyproject.toml b/integration_tests/iceberg-sink2/python/pyproject.toml index 4c7bce1165796..6a8e06b62215e 100644 --- a/integration_tests/iceberg-sink2/python/pyproject.toml +++ b/integration_tests/iceberg-sink2/python/pyproject.toml @@ -7,7 +7,7 @@ readme = "README.md" packages = [{include = "icelake_integration_tests"}] [tool.poetry.dependencies] -python = "^3.11" +python = "^3.8" pyspark = { version = "3.4.1", extras = ["sql", "connect"] } psycopg2-binary = "^2.9" diff --git a/integration_tests/iceberg-sink2/run.sh b/integration_tests/iceberg-sink2/run.sh new file mode 100755 index 0000000000000..d58973f6c7c8f --- /dev/null +++ b/integration_tests/iceberg-sink2/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +# Exits as soon as any line fails. +set -euox pipefail + +"$HOME"/.local/bin/poetry --version +cd python +# Don't remove the `--quiet` option since poetry has a bug when printing output, see +# https://github.com/python-poetry/poetry/issues/3412 +"$HOME"/.local/bin/poetry update --quiet +"$HOME"/.local/bin/poetry run python main.py \ No newline at end of file diff --git a/integration_tests/mysql-sink/create_sink.sql b/integration_tests/mysql-sink/create_sink.sql index bfe9bf6c0b70e..9776360df2914 100644 --- a/integration_tests/mysql-sink/create_sink.sql +++ b/integration_tests/mysql-sink/create_sink.sql @@ -17,3 +17,14 @@ FROM type = 'upsert', primary_key = 'id' ); + +CREATE SINK mysql_data_all_types_sink +FROM + mysql_all_types WITH ( + connector = 'jdbc', + jdbc.url = 'jdbc:mysql://mysql:3306/mydb?user=root&password=123456', + table.name = 'mysql_all_types', + type='append-only', + force_append_only = 'true', + primary_key = 'id' + ); diff --git a/integration_tests/mysql-sink/create_source.sql b/integration_tests/mysql-sink/create_source.sql index f049457aa3121..787d902ca3c95 100644 --- a/integration_tests/mysql-sink/create_source.sql +++ b/integration_tests/mysql-sink/create_source.sql @@ -40,3 +40,40 @@ VALUES (3, 'Varchar value 3', 'Text value 3', 345, 678, 901, 34.56, 78.90, 12.34, TRUE, '2023-05-24', '12:34:56', '2023-05-24 12:34:56', '2023-05-24T12:34:56Z', '{"key": "value3"}', E'\\xCAFEBABE'), (4, 'Varchar value 4', 'Text value 4', 456, 789, 012, 45.67, 89.01, 23.45, FALSE, '2023-05-25', '23:45:01', '2023-05-25 23:45:01', '2023-05-25T23:45:01Z', '{"key": "value4"}', E'\\xBABEC0DE'), (5, 'Varchar value 5', 'Text value 5', 567, 890, 123, 56.78, 90.12, 34.56, TRUE, '2023-05-26', '12:34:56', '2023-05-26 12:34:56', '2023-05-26T12:34:56Z', '{"key": "value5"}', E'\\xDEADBABE'); + + +CREATE TABLE mysql_all_types( + id integer PRIMARY KEY, + c_boolean boolean, + c_tinyint smallint, + c_smallint smallint, + c_mediumint integer, + c_integer integer, + c_bigint bigint, + c_decimal decimal, + c_float real, + c_double double, + c_char_255 varchar, + c_varchar_10000 varchar, + c_text varchar, + c_blob bytea, + c_binary_255 bytea, + c_varbinary_10000 bytea, + c_date date, + c_time time, + c_datetime timestamp, + c_timestamp timestamptz, + c_json JSONB, + c_smallint_array smallint[], + c_integer_array integer[], + c_bigint_array bigint[], + c_real_array real[], + c_double_precision_array double precision[], + c_varchar_array varchar[], +); + +INSERT INTO mysql_all_types VALUES (1, False, 0, 0, 0, 0, 0, 0, 0, 0, '', '', '', '\x00', '', '', '1001-01-01', '00:00:00.000000', '1000-01-01 00:00:00.000000', '1970-01-01 00:00:01.000000Z', '{}', array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::real[], array[]::double precision[], array[]::varchar[]); + +INSERT INTO mysql_all_types VALUES (2, False, -128, -32767, -8388608, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, 'a', 'b', 'c', '\x00', E'\\xFFABCE', '', '1970-01-01', '12:59:59.123456', '1000-01-01 00:00:00.000000', '2024-01-01 00:00:00.123456Z', '{"a": 1, "b":"c"}', array[-32767::smallint, 0]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array['aaa'::varchar]::varchar[]); + +INSERT INTO mysql_all_types VALUES (4, True, 127, 32767, 8388607, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', '\xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', '\xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', '\xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', '9999-12-31', '23:59:59.999999', '9999-12-31 23:59:59.499999', '2038-01-19 03:14:07.499999Z', '{"a": 1}', array[32767::smallint]::smallint[], array[2147483647::integer]::integer[], array[9223372036854775807::bigint]::bigint[], array[9999.999999::real]::real[], array[10000.0::double precision]::double precision[], array['zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'::varchar]::varchar[]); diff --git a/integration_tests/mysql-sink/mysql_prepare.sql b/integration_tests/mysql-sink/mysql_prepare.sql index d06e16e7829f5..a93a9de5a0e25 100644 --- a/integration_tests/mysql-sink/mysql_prepare.sql +++ b/integration_tests/mysql-sink/mysql_prepare.sql @@ -23,3 +23,33 @@ CREATE TABLE data_types ( jsonb_column JSON, bytea_column BLOB ); + +CREATE TABLE mysql_all_types ( + id integer PRIMARY KEY, + c_boolean boolean, + c_tinyint tinyint, + c_smallint smallint, + c_mediumint mediumint, + c_integer integer, + c_bigint bigint, + c_decimal decimal, + c_float float, + c_double double, + c_char_255 char(255), + c_varchar_10000 varchar(10000), + c_text text, + c_blob BLOB, + c_binary_255 binary(255), + c_varbinary_10000 varbinary(10000), + c_date date, + c_time time(6), + c_datetime datetime(6), + c_timestamp timestamp(6), + c_json JSON, + c_smallint_array LONGTEXT, + c_integer_array LONGTEXT, + c_bigint_array LONGTEXT, + c_real_array LONGTEXT, + c_double_precision_array LONGTEXT, + c_varchar_array LONGTEXT +); diff --git a/integration_tests/mysql-sink/prepare.sh b/integration_tests/mysql-sink/prepare.sh index 9f2e93d1b40a5..3374386d6aa90 100755 --- a/integration_tests/mysql-sink/prepare.sh +++ b/integration_tests/mysql-sink/prepare.sh @@ -2,7 +2,5 @@ set -euo pipefail -sleep 10 - # setup mysql docker compose exec mysql bash -c "mysql -p123456 -h mysql mydb < mysql_prepare.sql" diff --git a/integration_tests/mysql-sink/sink_check.py b/integration_tests/mysql-sink/sink_check.py index b7cf590c38d09..18d0d92a0f216 100644 --- a/integration_tests/mysql-sink/sink_check.py +++ b/integration_tests/mysql-sink/sink_check.py @@ -1,7 +1,7 @@ import subprocess import sys -relations = ['target_count', 'data_types'] +relations = ['target_count', 'data_types', 'mysql_all_types'] failed_cases = [] for rel in relations: diff --git a/integration_tests/nats/create_source.sql b/integration_tests/nats/create_source.sql index 0b15d9d859f6d..633f7a6d41bd1 100644 --- a/integration_tests/nats/create_source.sql +++ b/integration_tests/nats/create_source.sql @@ -1,21 +1,24 @@ -CREATE TABLE personnel (id integer, name varchar,); +CREATE TABLE + personnel (id integer, name varchar); -CREATE TABLE nats_source_table -( - id integer, - name varchar, -) -WITH ( - connector='nats', - server_url='nats-server:4222', - subject='subject1', + +CREATE TABLE + nats_source_table (id integer, name varchar) +WITH + ( + connector = 'nats', + server_url = 'nats-server:4222', + subject = 'subject1', stream = 'my_stream', - connect_mode='plain' -) FORMAT PLAIN ENCODE JSON; + connect_mode = 'plain' + ) FORMAT PLAIN ENCODE JSON; + CREATE SINK nats_sink FROM - personnel WITH ( + personnel +WITH + ( connector = 'nats', server_url = 'nats-server:4222', subject = 'subject1', @@ -24,28 +27,42 @@ FROM connect_mode = 'plain' ); -INSERT INTO - personnel -VALUES - (1, 'Alice'), - (2, 'Bob'); INSERT INTO personnel VALUES + (1, 'Alice'), + (2, 'Bob'), (3, 'Tom'), - (4, 'Jerry'); - -INSERT INTO - personnel -VALUES + (4, 'Jerry'), (5, 'Araminta'), - (6, 'Clover'); - -INSERT INTO - personnel -VALUES + (6, 'Clover'), (7, 'Posey'), (8, 'Waverly'); -FLUSH; \ No newline at end of file + +FLUSH; + + +CREATE TABLE live_stream_metrics ( + client_ip VARCHAR, + user_agent VARCHAR, + user_id VARCHAR, + room_id VARCHAR, + video_bps BIGINT, + video_fps BIGINT, + video_rtt BIGINT, + video_lost_pps BIGINT, + video_longest_freeze_duration BIGINT, + video_total_freeze_duration BIGINT, + report_timestamp TIMESTAMPTZ, + country VARCHAR +) +WITH + ( + connector = 'nats', + server_url = 'nats-server:4222', + subject = 'live_stream_metrics', + stream = 'risingwave', + connect_mode = 'plain' + ) FORMAT PLAIN ENCODE JSON; \ No newline at end of file diff --git a/integration_tests/nats/data_check b/integration_tests/nats/data_check index d95978e1a8734..27fd2594faf64 100644 --- a/integration_tests/nats/data_check +++ b/integration_tests/nats/data_check @@ -1 +1 @@ -personnel,nats_source_table \ No newline at end of file +personnel,nats_source_table,live_stream_metrics \ No newline at end of file diff --git a/integration_tests/nats/docker-compose.yml b/integration_tests/nats/docker-compose.yml index 7c70f79e0ab16..d1f8d747d3555 100644 --- a/integration_tests/nats/docker-compose.yml +++ b/integration_tests/nats/docker-compose.yml @@ -30,6 +30,15 @@ services: extends: file: ../../docker/docker-compose.yml service: message_queue + datagen: + build: ../datagen + depends_on: [message_queue] + command: + - /bin/sh + - -c + - /datagen --mode livestream --qps 10 nats --url nats-server:4222 --jetstream + restart: always + container_name: datagen volumes: compute-node-0: external: false diff --git a/integration_tests/nats/pb/create_source.sql b/integration_tests/nats/pb/create_source.sql new file mode 100644 index 0000000000000..b37c5634b00ad --- /dev/null +++ b/integration_tests/nats/pb/create_source.sql @@ -0,0 +1,12 @@ +CREATE TABLE live_stream_metrics +WITH + ( + connector = 'nats', + server_url = 'nats-server:4222', + subject = 'live_stream_metrics', + stream = 'risingwave', + connect_mode = 'plain' + ) FORMAT PLAIN ENCODE PROTOBUF ( + message = 'livestream.schema.LiveStreamMetrics', + schema.location = 'http://file_server:8080/schema' + ); \ No newline at end of file diff --git a/integration_tests/nats/schema b/integration_tests/nats/schema new file mode 100644 index 0000000000000..08b5cd4852c7e --- /dev/null +++ b/integration_tests/nats/schema @@ -0,0 +1,18 @@ + +‰ +livestream.protolivestream.schema"Å +LiveStreamMetrics + client_ip ( RclientIp + +user_agent ( R userAgent +user_id ( RuserId +room_id ( RroomId + video_bps (RvideoBps + video_fps (RvideoFps + video_rtt (RvideoRtt$ +video_lost_pps (R videoLostPpsA +video_longest_freeze_duration (RvideoLongestFreezeDuration= +video_total_freeze_duration + (RvideoTotalFreezeDuration) +report_timestamp (RreportTimestamp +country ( RcountryBZlivestream/protobproto3 \ No newline at end of file diff --git a/integration_tests/postgres-sink/create_sink.sql b/integration_tests/postgres-sink/create_sink.sql index e01ad2760cc2e..5041f1a36b741 100644 --- a/integration_tests/postgres-sink/create_sink.sql +++ b/integration_tests/postgres-sink/create_sink.sql @@ -18,3 +18,14 @@ FROM type='upsert', primary_key = 'id' ); + +CREATE SINK pg_all_data_types_sink +FROM + pg_all_data_types WITH ( + connector = 'jdbc', + jdbc.url = 'jdbc:postgresql://postgres:5432/mydb?user=myuser&password=123456', + table.name = 'pg_all_data_types', + type='append-only', + primary_key = 'id', + force_append_only = 'true', +); diff --git a/integration_tests/postgres-sink/create_source.sql b/integration_tests/postgres-sink/create_source.sql index 6840f8cb379c1..b480175c328b7 100644 --- a/integration_tests/postgres-sink/create_source.sql +++ b/integration_tests/postgres-sink/create_source.sql @@ -42,3 +42,34 @@ VALUES (3, 'Varchar value 3', 'Text value 3', 345, 678, 901, 34.56, 78.90, 12.34, TRUE, '2023-05-24', '12:34:56', '2023-05-24 12:34:56', '2023-05-24 12:34:56+00:00', '3 days', '{"key": "value3"}', E'\\xCAFEBABE', ARRAY['Value 5', 'Value 6']), (4, 'Varchar value 4', 'Text value 4', 456, 789, 012, 45.67, 89.01, 23.45, FALSE, '2023-05-25', '23:45:01', '2023-05-25 23:45:01', '2023-05-25 23:45:01+00:00', '4 days', '{"key": "value4"}', E'\\xBABEC0DE', ARRAY['Value 7', 'Value 8']), (5, 'Varchar value 5', 'Text value 5', 567, 890, 123, 56.78, 90.12, 34.56, TRUE, '2023-05-26', '12:34:56', '2023-05-26 12:34:56', '2023-05-26 12:34:56+00:00', '5 days', '{"key": "value5"}', E'\\xDEADBABE', ARRAY['Value 9', 'Value 10']); + +CREATE TABLE pg_all_data_types ( + id BIGINT PRIMARY KEY, + c_boolean boolean, + c_smallint smallint, + c_integer integer, + c_bigint bigint, + c_decimal decimal, + c_real real, + c_double_precision double precision, + c_varchar varchar, + c_bytea bytea, + c_date date, + c_time time, + c_timestamp timestamp, + c_timestamptz timestamptz, + c_interval interval, + c_jsonb jsonb, + c_smallint_array smallint[], + c_integer_array integer[], + c_bigint_array bigint[], + c_real_array real[], + c_double_precision_array double precision[], + c_varchar_array varchar[], +); + +INSERT INTO pg_all_data_types VALUES (1, False, 0, 0, 0, 0, 0, 0, '', '\x00', '0001-01-01', '00:00:00', '0001-01-01 00:00:00'::timestamp, '0001-01-01 00:00:00'::timestamptz, interval '0 second', '{}', array[]::smallint[], array[]::integer[], array[]::bigint[], array[]::real[], array[]::double precision[], array[]::varchar[]); + +INSERT INTO pg_all_data_types VALUES (2, False, -32767, -2147483647, -9223372036854775807, -10.0, -9999.999999, -10000.0, '', '\x00', '1970-01-01', '00:00:00.123456', '1970-01-01 00:00:00.123456', '1970-01-01 00:00:00.123456Z', interval '2 second', '{"a": 1, "b":"c"}', array[-32767::smallint, 0]::smallint[], array[-2147483647::integer]::integer[], array[-9223372036854775807::bigint]::bigint[], array[-9999.999999::real]::real[], array[-10000.0::double precision]::double precision[], array[''::varchar]::varchar[]); + +INSERT INTO pg_all_data_types VALUES (3, True, 32767, 2147483647, 9223372036854775807, -10.0, 9999.999999, 10000.0, 'zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz', '\xffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff', '9999-12-31', '23:59:59', '9999-12-31 23:59:59'::timestamp, '9999-12-31 23:59:59'::timestamptz, interval '9990 year', '{"whatever":"meaningless"}', array[32767::smallint]::smallint[], array[2147483647::integer]::integer[], array[9223372036854775807::bigint]::bigint[], array[9999.999999::real]::real[], array[10000.0::double precision]::double precision[], array['zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz'::varchar]::varchar[]); diff --git a/integration_tests/postgres-sink/postgres_prepare.sql b/integration_tests/postgres-sink/postgres_prepare.sql index 595cf960121d7..2d30f7d4fb357 100644 --- a/integration_tests/postgres-sink/postgres_prepare.sql +++ b/integration_tests/postgres-sink/postgres_prepare.sql @@ -22,4 +22,29 @@ CREATE TABLE data_types ( jsonb_column JSONB, bytea_column BYTEA, array_column VARCHAR[] -); \ No newline at end of file +); + +CREATE TABLE pg_all_data_types ( + id BIGINT PRIMARY KEY, + c_boolean boolean, + c_smallint smallint, + c_integer integer, + c_bigint bigint, + c_decimal decimal, + c_real real, + c_double_precision double precision, + c_varchar varchar, + c_bytea bytea, + c_date date, + c_time time, + c_timestamp timestamp, + c_timestamptz timestamptz, + c_interval interval, + c_jsonb jsonb, + c_smallint_array smallint[], + c_integer_array integer[], + c_bigint_array bigint[], + c_real_array real[], + c_double_precision_array double precision[], + c_varchar_array varchar[] +); diff --git a/integration_tests/postgres-sink/sink_check.py b/integration_tests/postgres-sink/sink_check.py index 606b78424a262..ec6d607f0e9b2 100644 --- a/integration_tests/postgres-sink/sink_check.py +++ b/integration_tests/postgres-sink/sink_check.py @@ -2,7 +2,7 @@ import subprocess -relations = ['target_count', 'data_types'] +relations = ['target_count', 'data_types', 'pg_all_data_types'] failed_cases = [] for rel in relations: diff --git a/integration_tests/scripts/check_data.py b/integration_tests/scripts/check_data.py index 9c449d5bff43b..62b887e1b724e 100644 --- a/integration_tests/scripts/check_data.py +++ b/integration_tests/scripts/check_data.py @@ -109,7 +109,7 @@ def test_check(demo: str, upstream: str, need_data_check=True, need_sink_check=F demo = sys.argv[1] upstream = sys.argv[2] # mysql, postgres, etc. see scripts/integration_tests.sh -if demo in ['docker', 'iceberg-cdc']: +if demo in ['docker', 'iceberg-cdc', 'iceberg-sink']: print('Skip for running test for `%s`' % demo) sys.exit(0) diff --git a/integration_tests/scripts/clean_demos.py b/integration_tests/scripts/clean_demos.py index c396a9c94d8b0..2a884832395fa 100644 --- a/integration_tests/scripts/clean_demos.py +++ b/integration_tests/scripts/clean_demos.py @@ -4,6 +4,7 @@ import os import subprocess import argparse +import sys def clean_demo(demo: str): @@ -22,5 +23,10 @@ def clean_demo(demo: str): type=str, help='the test case') args = arg_parser.parse_args() +demo = args.case -clean_demo(args.case) +if demo in ['iceberg-sink']: + print('Skip for running test for `%s`' % demo) + sys.exit(0) + +clean_demo(demo) diff --git a/integration_tests/scripts/run_demos.py b/integration_tests/scripts/run_demos.py index 87967daa50b2c..d6fd4ecbed9c3 100644 --- a/integration_tests/scripts/run_demos.py +++ b/integration_tests/scripts/run_demos.py @@ -42,7 +42,7 @@ def run_demo(demo: str, format: str, wait_time=40): sleep(10) continue # Fallback to default version when the protobuf version doesn't exist. - sql_file = os.path.join(demo_dir, fname) + sql_file = os.path.join(demo_dir, fname) if not os.path.exists(sql_file): continue run_sql_file(sql_file, demo_dir) @@ -55,7 +55,16 @@ def iceberg_cdc_demo(): project_dir = dirname(file_dir) demo_dir = os.path.join(project_dir, demo) print("Running demo: iceberg-cdc") - subprocess.run(["bash","./run_test.sh"], cwd=demo_dir, check=True) + subprocess.run(["bash", "./run_test.sh"], cwd=demo_dir, check=True) + + +def iceberg_sink_demo(): + demo = "iceberg-sink2" + file_dir = dirname(abspath(__file__)) + project_dir = dirname(file_dir) + demo_dir = os.path.join(project_dir, demo) + print("Running demo: iceberg-sink2") + subprocess.run(["bash", "./run.sh"], cwd=demo_dir, check=True) arg_parser = argparse.ArgumentParser(description="Run the demo") @@ -75,5 +84,7 @@ def iceberg_cdc_demo(): if args.case == "iceberg-cdc": iceberg_cdc_demo() +elif args.case == "iceberg-sink": + iceberg_sink_demo() else: run_demo(args.case, args.format) diff --git a/integration_tests/starrocks-sink/README.md b/integration_tests/starrocks-sink/README.md index 817ab57481e43..660fcf962a6d9 100644 --- a/integration_tests/starrocks-sink/README.md +++ b/integration_tests/starrocks-sink/README.md @@ -18,12 +18,12 @@ Login to mysql docker compose exec starrocks-fe mysql -uroot -P9030 -h127.0.0.1 ``` -Run the following queries to create database and table. +Run the following queries to create database and table. You also use other starrocks table, example in ./starrocks_prepare.sql ```sql CREATE database demo; use demo; -CREATE table demo_bhv_table( +CREATE table upsert_table( user_id int, target_id text, event_timestamp_local datetime @@ -37,20 +37,14 @@ GRANT ALL ON *.* TO 'users'@'%'; 3. Execute the SQL queries in sequence: -- append-only sql: - - create_source.sql - - create_mv.sql - - create_sink.sql - -- upsert sql: - - upsert/create_table.sql - - upsert/create_mv.sql - - upsert/create_sink.sql - - upsert/insert_update_delete.sql +- create_source.sql +- create_mv.sql +- create_sink.sql +- update_delete.sql We only support `upsert` with starrocks' `PRIMARY KEY` Run the following query ```sql -select user_id, count(*) from demo.demo_bhv_table group by user_id; +select user_id, count(*) from demo.upsert_table group by user_id; ``` diff --git a/integration_tests/starrocks-sink/create_mv.sql b/integration_tests/starrocks-sink/create_mv.sql index c367e6f2baa94..6e466703b0769 100644 --- a/integration_tests/starrocks-sink/create_mv.sql +++ b/integration_tests/starrocks-sink/create_mv.sql @@ -5,3 +5,11 @@ SELECT event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local FROM user_behaviors; + +CREATE MATERIALIZED VIEW upsert_bhv_mv AS +SELECT + user_id, + target_id, + event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local +FROM + upsert_user_behaviors; diff --git a/integration_tests/starrocks-sink/create_sink.sql b/integration_tests/starrocks-sink/create_sink.sql index 56d1b227512de..5cd98ba1ec17f 100644 --- a/integration_tests/starrocks-sink/create_sink.sql +++ b/integration_tests/starrocks-sink/create_sink.sql @@ -1,4 +1,4 @@ -CREATE SINK bhv_starrocks_sink +CREATE SINK bhv_starrocks_sink_primary FROM bhv_mv WITH ( connector = 'starrocks', @@ -9,6 +9,66 @@ FROM starrocks.user = 'users', starrocks.password = '123456', starrocks.database = 'demo', - starrocks.table = 'demo_bhv_table', + starrocks.table = 'demo_primary_table', force_append_only='true' -); \ No newline at end of file +); + +CREATE SINK bhv_starrocks_sink_duplicate +FROM + bhv_mv WITH ( + connector = 'starrocks', + type = 'append-only', + starrocks.host = 'starrocks-fe', + starrocks.mysqlport = '9030', + starrocks.httpport = '8030', + starrocks.user = 'users', + starrocks.password = '123456', + starrocks.database = 'demo', + starrocks.table = 'demo_duplicate_table', + force_append_only='true' +); + +CREATE SINK bhv_starrocks_sink_aggregate +FROM + bhv_mv WITH ( + connector = 'starrocks', + type = 'append-only', + starrocks.host = 'starrocks-fe', + starrocks.mysqlport = '9030', + starrocks.httpport = '8030', + starrocks.user = 'users', + starrocks.password = '123456', + starrocks.database = 'demo', + starrocks.table = 'demo_aggregate_table', + force_append_only='true' +); + +CREATE SINK bhv_starrocks_sink_unique +FROM + bhv_mv WITH ( + connector = 'starrocks', + type = 'append-only', + starrocks.host = 'starrocks-fe', + starrocks.mysqlport = '9030', + starrocks.httpport = '8030', + starrocks.user = 'users', + starrocks.password = '123456', + starrocks.database = 'demo', + starrocks.table = 'demo_unique_table', + force_append_only='true' +); + +CREATE SINK upsert_starrocks_sink +FROM + upsert_bhv_mv WITH ( + connector = 'starrocks', + type = 'upsert', + starrocks.host = 'starrocks-fe', + starrocks.mysqlport = '9030', + starrocks.httpport = '8030', + starrocks.user = 'users', + starrocks.password = '123456', + starrocks.database = 'demo', + starrocks.table = 'upsert_table', + primary_key = 'user_id' +); diff --git a/integration_tests/starrocks-sink/create_source.sql b/integration_tests/starrocks-sink/create_source.sql index ed7c02341638a..0e42308511121 100644 --- a/integration_tests/starrocks-sink/create_source.sql +++ b/integration_tests/starrocks-sink/create_source.sql @@ -14,3 +14,20 @@ CREATE table user_behaviors ( fields.user_id.end = '1000', datagen.rows.per.second = '100' ) FORMAT PLAIN ENCODE JSON; + +CREATE table upsert_user_behaviors ( + user_id int, + target_id VARCHAR, + target_type VARCHAR, + event_timestamp TIMESTAMPTZ, + behavior_type VARCHAR, + parent_target_type VARCHAR, + parent_target_id VARCHAR, + PRIMARY KEY(user_id) +); + +INSERT INTO upsert_user_behaviors VALUES + (1,'1','1','2020-01-01T01:01:01Z','1','1','1'), + (2,'2','2','2020-01-01T01:01:02Z','2','2','2'), + (3,'3','3','2020-01-01T01:01:03Z','3','3','3'), + (4,'4','4','2020-01-01T01:01:04Z','4','4','4'); diff --git a/integration_tests/starrocks-sink/docker-compose.yml b/integration_tests/starrocks-sink/docker-compose.yml index 41dabac20dc7f..81ef7c277dad0 100644 --- a/integration_tests/starrocks-sink/docker-compose.yml +++ b/integration_tests/starrocks-sink/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: starrocks-fe: - image: starrocks/fe-ubuntu:latest + image: starrocks/fe-ubuntu:3.1.7 hostname: starrocks-fe container_name: starrocks-fe volumes: @@ -19,7 +19,7 @@ services: timeout: 5s retries: 30 starrocks-be: - image: starrocks/be-ubuntu:latest + image: starrocks/be-ubuntu:3.1.7 command: - /bin/bash - -c @@ -27,6 +27,7 @@ services: sleep 15s; mysql --connect-timeout 2 -h starrocks-fe -P9030 -uroot -e "alter system add backend \"starrocks-be:9050\";" /opt/starrocks/be/bin/start_be.sh ports: + - 9050:9050 - 8040:8040 hostname: starrocks-be container_name: starrocks-be @@ -52,6 +53,12 @@ services: extends: file: ../../docker/docker-compose.yml service: prometheus-0 + postgres: + image: postgres:latest + command: tail -f /dev/null + volumes: + - "./update_delete.sql:/update_delete.sql" + restart: on-failure volumes: risingwave-standalone: external: false diff --git a/integration_tests/starrocks-sink/sink_check.py b/integration_tests/starrocks-sink/sink_check.py index 699304854dc1f..bbe76e02a48de 100644 --- a/integration_tests/starrocks-sink/sink_check.py +++ b/integration_tests/starrocks-sink/sink_check.py @@ -1,7 +1,7 @@ import subprocess import sys -relations = ['demo.demo_bhv_table'] +relations = ['demo.demo_primary_table','demo.demo_duplicate_table','demo.demo_aggregate_table','demo.demo_unique_table', 'demo.upsert_table'] failed_cases = [] for rel in relations: @@ -18,6 +18,30 @@ if rows < 1: failed_cases.append(rel) +# update data +subprocess.run(["docker", "compose", "exec", "postgres", "bash", "-c", "psql -h risingwave-standalone -p 4566 -d dev -U root -f update_delete.sql"], check=True) + +# delete +sql = f"SELECT COUNT(*) FROM demo.upsert_table;" +command = f'mysql -uroot -P9030 -h127.0.0.1 -e "{sql}"' +output = subprocess.check_output( + ["docker", "compose", "exec", "starrocks-fe", "bash", "-c", command]) +rows = int(output.decode('utf-8').split('\n')[1]) +print(f"{rows} rows in demo.upsert_table") +if rows != 3: + print(f"rows expected 3, get {rows}") + failed_cases.append("delete demo.upsert_table") + +# update +sql = f"SELECT target_id FROM demo.upsert_table WHERE user_id = 3;" +command = f'mysql -uroot -P9030 -h127.0.0.1 -e "{sql}"' +output = subprocess.check_output( + ["docker", "compose", "exec", "starrocks-fe", "bash", "-c", command]) +id = int(output.decode('utf-8').split('\n')[1]) +if id != 30: + print(f"target_id expected 30, get {id}") + failed_cases.append("update demo.upsert_table") + if len(failed_cases) != 0: print(f"Data check failed for case {failed_cases}") sys.exit(1) diff --git a/integration_tests/starrocks-sink/starrocks_prepare.sql b/integration_tests/starrocks-sink/starrocks_prepare.sql index aadaf85289b3c..7556b6db1dcf4 100644 --- a/integration_tests/starrocks-sink/starrocks_prepare.sql +++ b/integration_tests/starrocks-sink/starrocks_prepare.sql @@ -1,7 +1,7 @@ CREATE database demo; use demo; -CREATE table demo_bhv_table( +CREATE table demo_primary_table( user_id int, target_id text, event_timestamp_local datetime @@ -9,5 +9,37 @@ CREATE table demo_bhv_table( PRIMARY KEY(`user_id`) DISTRIBUTED BY HASH(`user_id`) properties("replication_num" = "1"); +CREATE table upsert_table( + user_id int, + target_id text, + event_timestamp_local datetime +) ENGINE=OLAP +PRIMARY KEY(`user_id`) +DISTRIBUTED BY HASH(`user_id`) properties("replication_num" = "1"); + +CREATE table demo_duplicate_table( + user_id int, + target_id text, + event_timestamp_local datetime +) ENGINE=OLAP +DUPLICATE KEY(`user_id`) +DISTRIBUTED BY HASH(`user_id`) properties("replication_num" = "1"); + +CREATE table demo_aggregate_table( + user_id int, + target_id text, + event_timestamp_local datetime +) ENGINE=OLAP +AGGREGATE KEY(`user_id`,`target_id`,`event_timestamp_local`) +DISTRIBUTED BY HASH(`user_id`) properties("replication_num" = "1"); + +CREATE table demo_unique_table( + user_id int, + target_id text, + event_timestamp_local datetime +) ENGINE=OLAP +UNIQUE KEY(`user_id`) +DISTRIBUTED BY HASH(`user_id`) properties("replication_num" = "1"); + CREATE USER 'users'@'%' IDENTIFIED BY '123456'; GRANT ALL ON *.* TO 'users'@'%'; diff --git a/integration_tests/starrocks-sink/update_delete.sql b/integration_tests/starrocks-sink/update_delete.sql new file mode 100644 index 0000000000000..adabd5163ef44 --- /dev/null +++ b/integration_tests/starrocks-sink/update_delete.sql @@ -0,0 +1,5 @@ +DELETE FROM upsert_user_behaviors WHERE user_id = 2; + +UPDATE upsert_user_behaviors SET target_id = 30 WHERE user_id = 3; + +FLUSH; diff --git a/integration_tests/starrocks-sink/upsert/create_mv.sql b/integration_tests/starrocks-sink/upsert/create_mv.sql deleted file mode 100644 index c367e6f2baa94..0000000000000 --- a/integration_tests/starrocks-sink/upsert/create_mv.sql +++ /dev/null @@ -1,7 +0,0 @@ -CREATE MATERIALIZED VIEW bhv_mv AS -SELECT - user_id, - target_id, - event_timestamp AT TIME ZONE 'Asia/Shanghai' as event_timestamp_local -FROM - user_behaviors; diff --git a/integration_tests/starrocks-sink/upsert/create_sink.sql b/integration_tests/starrocks-sink/upsert/create_sink.sql deleted file mode 100644 index d7557bc1bd4fc..0000000000000 --- a/integration_tests/starrocks-sink/upsert/create_sink.sql +++ /dev/null @@ -1,14 +0,0 @@ -CREATE SINK bhv_starrocks_sink -FROM - bhv_mv WITH ( - connector = 'starrocks', - type = 'upsert', - starrocks.host = 'starrocks-fe', - starrocks.mysqlport = '9030', - starrocks.httpport = '8030', - starrocks.user = 'users', - starrocks.password = '123456', - starrocks.database = 'demo', - starrocks.table = 'demo_bhv_table', - primary_key = 'user_id' -); \ No newline at end of file diff --git a/integration_tests/starrocks-sink/upsert/create_table.sql b/integration_tests/starrocks-sink/upsert/create_table.sql deleted file mode 100644 index c6cfa87eed3c8..0000000000000 --- a/integration_tests/starrocks-sink/upsert/create_table.sql +++ /dev/null @@ -1,10 +0,0 @@ -CREATE table user_behaviors ( - user_id int, - target_id VARCHAR, - target_type VARCHAR, - event_timestamp TIMESTAMPTZ, - behavior_type VARCHAR, - parent_target_type VARCHAR, - parent_target_id VARCHAR, - PRIMARY KEY(user_id) -); diff --git a/integration_tests/starrocks-sink/upsert/insert_update_delete.sql b/integration_tests/starrocks-sink/upsert/insert_update_delete.sql deleted file mode 100644 index f21353c161154..0000000000000 --- a/integration_tests/starrocks-sink/upsert/insert_update_delete.sql +++ /dev/null @@ -1,8 +0,0 @@ -INSERT INTO user_behaviors VALUES(1,'1','1','2020-01-01T01:01:01Z','1','1','1'), -(2,'2','2','2020-01-01T01:01:02Z','2','2','2'), -(3,'3','3','2020-01-01T01:01:03Z','3','3','3'), -(4,'4','4','2020-01-01T01:01:04Z','4','4','4'); - -DELETE FROM user_behaviors WHERE user_id = 2; - -UPDATE user_behaviors SET target_id = 30 WHERE user_id = 3; diff --git a/java/connector-node/assembly/assembly.xml b/java/connector-node/assembly/assembly.xml index 9cf457d8a0b6d..26df6e8a71af9 100644 --- a/java/connector-node/assembly/assembly.xml +++ b/java/connector-node/assembly/assembly.xml @@ -42,9 +42,13 @@ *:risingwave-sink-es-7 *:risingwave-sink-cassandra *:risingwave-sink-jdbc + *:risingwave-sink-iceberg *:risingwave-sink-mock-flink-http-sink + + org.apache.iceberg:iceberg-common + true true diff --git a/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java index 88b8351f80fb4..0c9858ab4fd5d 100644 --- a/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java +++ b/java/connector-node/connector-api/src/main/java/com/risingwave/connector/api/source/SourceTypeE.java @@ -20,6 +20,7 @@ public enum SourceTypeE { MYSQL, POSTGRES, CITUS, + MONGODB, INVALID; public static SourceTypeE valueOf(ConnectorServiceProto.SourceType type) { @@ -30,6 +31,8 @@ public static SourceTypeE valueOf(ConnectorServiceProto.SourceType type) { return SourceTypeE.POSTGRES; case CITUS: return SourceTypeE.CITUS; + case MONGODB: + return SourceTypeE.MONGODB; default: return SourceTypeE.INVALID; } diff --git a/java/connector-node/risingwave-connector-service/pom.xml b/java/connector-node/risingwave-connector-service/pom.xml index 047c523c1c7db..d51d67497ce05 100644 --- a/java/connector-node/risingwave-connector-service/pom.xml +++ b/java/connector-node/risingwave-connector-service/pom.xml @@ -99,7 +99,6 @@ com.risingwave risingwave-sink-mock-flink-http-sink - provided diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java index 296592fbd784c..1f1c68f54551e 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/JniSinkWriterResponseObserver.java @@ -41,11 +41,12 @@ public void onNext(ConnectorServiceProto.SinkWriterStreamResponse response) { @Override public void onError(Throwable throwable) { - if (!Binding.sendSinkWriterErrorToChannel(this.responseTxPtr, throwable.getMessage())) { + LOG.error("JniSinkWriterHandler onError: ", throwable); + var errMsg = throwable.getMessage() == null ? "unknown error" : throwable.getMessage(); + if (!Binding.sendSinkWriterErrorToChannel(this.responseTxPtr, errMsg)) { LOG.warn("unable to send error: {}", throwable.getMessage()); } this.success = false; - LOG.error("JniSinkWriterHandler onError: ", throwable); } @Override diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java index 3ef406f97754f..e76a8b59facc0 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/SourceValidateHandler.java @@ -62,33 +62,43 @@ public static ConnectorServiceProto.ValidateSourceResponse validateResponse(Stri .build(); } - public static void ensurePropNotBlank(Map props, String name) { + private static void ensurePropNotBlank(Map props, String name) { + ensurePropsExists(props, name); if (StringUtils.isBlank(props.get(name))) { throw ValidatorUtils.invalidArgument( - String.format("'%s' not found, please check the WITH properties", name)); + String.format("'%s' cannot be empty. Please check the WITH properties", name)); } } - public static void validateSource(ConnectorServiceProto.ValidateSourceRequest request) - throws Exception { - var props = request.getPropertiesMap(); + private static void ensurePropsExists(Map props, String name) { + if (!props.containsKey(name)) { + throw ValidatorUtils.invalidArgument( + String.format("'%s' is not found. Please check the WITH properties", name)); + } + } + static void ensureRequiredProps(Map props, boolean isMultiTableShared) { ensurePropNotBlank(props, DbzConnectorConfig.HOST); ensurePropNotBlank(props, DbzConnectorConfig.PORT); ensurePropNotBlank(props, DbzConnectorConfig.DB_NAME); ensurePropNotBlank(props, DbzConnectorConfig.USER); - ensurePropNotBlank(props, DbzConnectorConfig.PASSWORD); - - var commonParam = request.getCommonParam(); - boolean isMultiTableShared = commonParam.getIsMultiTableShared(); + ensurePropsExists(props, DbzConnectorConfig.PASSWORD); // ensure table name is passed by user in non-sharing mode if (!isMultiTableShared) { ensurePropNotBlank(props, DbzConnectorConfig.TABLE_NAME); } + } + + public static void validateSource(ConnectorServiceProto.ValidateSourceRequest request) + throws Exception { + var props = request.getPropertiesMap(); + var commonParam = request.getCommonParam(); + boolean isMultiTableShared = commonParam.getIsMultiTableShared(); TableSchema tableSchema = TableSchema.fromProto(request.getTableSchema()); switch (request.getSourceType()) { case POSTGRES: + ensureRequiredProps(props, isMultiTableShared); ensurePropNotBlank(props, DbzConnectorConfig.PG_SCHEMA_NAME); ensurePropNotBlank(props, DbzConnectorConfig.PG_SLOT_NAME); ensurePropNotBlank(props, DbzConnectorConfig.PG_PUB_NAME); @@ -100,6 +110,7 @@ public static void validateSource(ConnectorServiceProto.ValidateSourceRequest re break; case CITUS: + ensureRequiredProps(props, isMultiTableShared); ensurePropNotBlank(props, DbzConnectorConfig.TABLE_NAME); ensurePropNotBlank(props, DbzConnectorConfig.PG_SCHEMA_NAME); try (var coordinatorValidator = new CitusValidator(props, tableSchema)) { @@ -128,11 +139,18 @@ public static void validateSource(ConnectorServiceProto.ValidateSourceRequest re break; case MYSQL: + ensureRequiredProps(props, isMultiTableShared); ensurePropNotBlank(props, DbzConnectorConfig.MYSQL_SERVER_ID); try (var validator = new MySqlValidator(props, tableSchema)) { validator.validateAll(isMultiTableShared); } break; + case MONGODB: + ensurePropNotBlank(props, DbzConnectorConfig.MongoDb.MONGO_URL); + ensurePropNotBlank(props, DbzConnectorConfig.MongoDb.MONGO_COLLECTION_NAME); + var validator = new MongoDbValidator(props); + validator.validateDbConfig(); + break; default: LOG.warn("Unknown source type"); throw ValidatorUtils.invalidArgument("Unknown source type"); diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java index b796cae4ccb4a..5406f6fd5e952 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/DbzConnectorConfig.java @@ -14,6 +14,7 @@ package com.risingwave.connector.source.common; +import com.mongodb.ConnectionString; import com.risingwave.connector.api.source.SourceTypeE; import com.risingwave.connector.cdc.debezium.internal.ConfigurableOffsetBackingStore; import java.io.IOException; @@ -61,12 +62,18 @@ public class DbzConnectorConfig { private static final String DBZ_CONFIG_FILE = "debezium.properties"; private static final String MYSQL_CONFIG_FILE = "mysql.properties"; private static final String POSTGRES_CONFIG_FILE = "postgres.properties"; + private static final String MONGODB_CONFIG_FILE = "mongodb.properties"; private static final String DBZ_PROPERTY_PREFIX = "debezium."; private static final String SNAPSHOT_MODE_KEY = "debezium.snapshot.mode"; private static final String SNAPSHOT_MODE_BACKFILL = "rw_cdc_backfill"; + public static class MongoDb { + public static final String MONGO_URL = "mongodb.url"; + public static final String MONGO_COLLECTION_NAME = "collection.name"; + } + private static Map extractDebeziumProperties( Map userProperties) { // retain only debezium properties if any @@ -217,6 +224,27 @@ public DbzConnectorConfig( ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset); } dbzProps.putAll(postgresProps); + } else if (source == SourceTypeE.MONGODB) { + var mongodbProps = initiateDbConfig(MONGODB_CONFIG_FILE, substitutor); + + // if snapshot phase is finished and offset is specified, we will continue reading + // changes from the given offset + if (snapshotDone && null != startOffset && !startOffset.isBlank()) { + mongodbProps.setProperty("snapshot.mode", "never"); + mongodbProps.setProperty( + ConfigurableOffsetBackingStore.OFFSET_STATE_VALUE, startOffset); + } + + var mongodbUrl = userProps.get("mongodb.url"); + var collection = userProps.get("collection.name"); + var connectionStr = new ConnectionString(mongodbUrl); + var connectorName = + String.format( + "MongoDB_%d:%s:%s", sourceId, connectionStr.getHosts(), collection); + mongodbProps.setProperty("name", connectorName); + + dbzProps.putAll(mongodbProps); + } else { throw new RuntimeException("unsupported source type: " + source); } diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MongoDbValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MongoDbValidator.java new file mode 100644 index 0000000000000..0ce5634e34b46 --- /dev/null +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/MongoDbValidator.java @@ -0,0 +1,56 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.risingwave.connector.source.common; + +import com.mongodb.client.MongoClient; +import com.mongodb.client.MongoClients; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class MongoDbValidator extends DatabaseValidator { + private static final Logger LOG = LoggerFactory.getLogger(MongoDbValidator.class); + + String mongodbUrl; + + public MongoDbValidator(Map userProps) { + this.mongodbUrl = userProps.get("mongodb.url"); + } + + @Override + public void validateDbConfig() { + // check connectivity + try (MongoClient mongoClient = MongoClients.create(mongodbUrl)) { + var desc = mongoClient.getClusterDescription(); + LOG.info("test connectivity: MongoDB cluster description: {}", desc); + } + } + + @Override + void validateUserPrivilege() { + // TODO: check user privilege + // https://debezium.io/documentation/reference/stable/connectors/mongodb.html#setting-up-mongodb + // You must also have a MongoDB user that has the appropriate roles to read the admin + // database where the oplog can be read. Additionally, the user must also be able to read + // the config database in the configuration server of a sharded cluster and must have + // listDatabases privilege action. When change streams are used (the default) the user also + // must have cluster-wide privilege actions find and changeStream. + } + + @Override + void validateTable() { + // do nothing since MongoDB is schemaless + } +} diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java index f8a63971b95e4..7014c3adb92da 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/PostgresValidator.java @@ -42,7 +42,7 @@ public class PostgresValidator extends DatabaseValidator implements AutoCloseabl private final boolean pubAutoCreate; - private static final String AWS_RDS_HOST = "rds.amazonaws.com"; + private static final String AWS_RDS_HOST = "amazonaws.com"; private final boolean isAwsRds; // Whether the properties to validate is shared by multiple tables. diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java index 8bb5cc0de3487..2b55adc44a306 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEngineRunner.java @@ -17,6 +17,7 @@ import com.risingwave.connector.api.source.*; import com.risingwave.connector.source.common.DbzConnectorConfig; import com.risingwave.connector.source.common.DbzSourceUtils; +import com.risingwave.java.binding.Binding; import com.risingwave.proto.ConnectorServiceProto.GetEventStreamResponse; import io.debezium.config.CommonConnectorConfig; import io.grpc.stub.StreamObserver; @@ -32,7 +33,7 @@ public class DbzCdcEngineRunner implements CdcEngineRunner { private final ExecutorService executor; private final AtomicBoolean running = new AtomicBoolean(false); - private final CdcEngine engine; + private CdcEngine engine; private final DbzConnectorConfig config; public static CdcEngineRunner newCdcEngineRunner( @@ -46,29 +47,33 @@ public static CdcEngineRunner newCdcEngineRunner( config.getResolvedDebeziumProps(), (success, message, error) -> { if (!success) { - responseObserver.onError(error); LOG.error( "engine#{} terminated with error. message: {}", sourceId, message, error); + if (error != null) { + responseObserver.onError(error); + } } else { LOG.info("engine#{} stopped normally. {}", sourceId, message); responseObserver.onCompleted(); } }); - runner = new DbzCdcEngineRunner(engine, config); + runner = new DbzCdcEngineRunner(config); + runner.withEngine(engine); } catch (Exception e) { LOG.error("failed to create the CDC engine", e); } return runner; } - public static CdcEngineRunner create(DbzConnectorConfig config) { - DbzCdcEngineRunner runner = null; + public static CdcEngineRunner create(DbzConnectorConfig config, long channelPtr) { + DbzCdcEngineRunner runner = new DbzCdcEngineRunner(config); try { var sourceId = config.getSourceId(); + final DbzCdcEngineRunner finalRunner = runner; var engine = new DbzCdcEngine( config.getSourceId(), @@ -80,27 +85,46 @@ public static CdcEngineRunner create(DbzConnectorConfig config) { sourceId, message, error); + String errorMsg = + (error != null ? error.getMessage() : message); + if (!Binding.sendCdcSourceErrorToChannel( + channelPtr, errorMsg)) { + LOG.warn( + "engine#{} unable to send error message: {}", + sourceId, + errorMsg); + } + // We need to stop the engine runner on debezium engine failure + try { + finalRunner.stop(); + } catch (Exception e) { + LOG.warn("failed to stop the engine#{}", sourceId, e); + } } else { LOG.info("engine#{} stopped normally. {}", sourceId, message); } }); - runner = new DbzCdcEngineRunner(engine, config); + runner.withEngine(engine); } catch (Exception e) { LOG.error("failed to create the CDC engine", e); + runner = null; } return runner; } // private constructor - private DbzCdcEngineRunner(CdcEngine engine, DbzConnectorConfig config) { + private DbzCdcEngineRunner(DbzConnectorConfig config) { this.executor = Executors.newSingleThreadExecutor( - r -> new Thread(r, "rw-dbz-engine-runner-" + engine.getId())); - this.engine = engine; + r -> new Thread(r, "rw-dbz-engine-runner-" + config.getSourceId())); this.config = config; } + private void withEngine(CdcEngine engine) { + this.engine = engine; + } + /** Start to run the cdc engine */ public boolean start() throws InterruptedException { if (isRunning()) { diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEventConsumer.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEventConsumer.java index f0880d52c8b57..bcc532038c9c6 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEventConsumer.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/DbzCdcEventConsumer.java @@ -46,7 +46,8 @@ public class DbzCdcEventConsumer private final BlockingQueue outputChannel; private final long sourceId; - private final JsonConverter converter; + private final JsonConverter payloadConverter; + private final JsonConverter keyConverter; private final String heartbeatTopicPrefix; private final String transactionTopic; @@ -64,14 +65,19 @@ public class DbzCdcEventConsumer // The default JSON converter will output the schema field in the JSON which is unnecessary // to source parser, we use a customized JSON converter to avoid outputting the `schema` // field. - var jsonConverter = new DbzJsonConverter(); + var payloadConverter = new DbzJsonConverter(); final HashMap configs = new HashMap<>(2); // only serialize the value part configs.put(ConverterConfig.TYPE_CONFIG, ConverterType.VALUE.getName()); // include record schema to output JSON in { "schema": { ... }, "payload": { ... } } format configs.put(JsonConverterConfig.SCHEMAS_ENABLE_CONFIG, true); - jsonConverter.configure(configs); - this.converter = jsonConverter; + payloadConverter.configure(configs); + this.payloadConverter = payloadConverter; + + var keyConverter = new DbzJsonConverter(); + configs.put(ConverterConfig.TYPE_CONFIG, ConverterType.KEY.getName()); + keyConverter.configure(configs); + this.keyConverter = keyConverter; } private EventType getEventType(SourceRecord record) { @@ -137,7 +143,7 @@ var record = event.value(); { long trxTs = ((Struct) record.value()).getInt64("ts_ms"); byte[] payload = - converter.fromConnectData( + payloadConverter.fromConnectData( record.topic(), record.valueSchema(), record.value()); var message = msgBuilder @@ -152,8 +158,9 @@ var record = event.value(); case DATA: { // Topic naming conventions - // - PG: serverName.schemaName.tableName - // - MySQL: serverName.databaseName.tableName + // - PG: topicPrefix.schemaName.tableName + // - MySQL: topicPrefix.databaseName.tableName + // - Mongo: topicPrefix.databaseName.collectionName // We can extract the full table name from the topic var fullTableName = record.topic().substring(record.topic().indexOf('.') + 1); @@ -169,15 +176,23 @@ var record = event.value(); ? System.currentTimeMillis() : sourceStruct.getInt64("ts_ms"); byte[] payload = - converter.fromConnectData( + payloadConverter.fromConnectData( record.topic(), record.valueSchema(), record.value()); + byte[] key = + keyConverter.fromConnectData( + record.topic(), record.keySchema(), record.key()); var message = msgBuilder .setFullTableName(fullTableName) .setPayload(new String(payload, StandardCharsets.UTF_8)) + .setKey(new String(key, StandardCharsets.UTF_8)) .setSourceTsMs(sourceTsMs) .build(); - LOG.debug("record => {}", message.getPayload()); + LOG.debug( + "offset => {}, key => {}, payload => {}", + message.getOffset(), + message.getKey(), + message.getPayload()); respBuilder.addEvents(message); break; } @@ -189,6 +204,7 @@ var record = event.value(); committer.markProcessed(event); } + LOG.debug("recv {} events", respBuilder.getEventsCount()); // skip empty batch if (respBuilder.getEventsCount() > 0) { respBuilder.setSourceId(sourceId); diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/JniDbzSourceHandler.java b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/JniDbzSourceHandler.java index d8ed1e8045a88..00b387eff2bb0 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/JniDbzSourceHandler.java +++ b/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/core/JniDbzSourceHandler.java @@ -72,7 +72,7 @@ public static void runJniDbzSourceThread(byte[] getEventStreamRequestBytes, long } public void start(long channelPtr) { - var runner = DbzCdcEngineRunner.create(config); + var runner = DbzCdcEngineRunner.create(config, channelPtr); if (runner == null) { return; } diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/mongodb.properties b/java/connector-node/risingwave-connector-service/src/main/resources/mongodb.properties new file mode 100644 index 0000000000000..a1a5b92e9f63f --- /dev/null +++ b/java/connector-node/risingwave-connector-service/src/main/resources/mongodb.properties @@ -0,0 +1,15 @@ +# configs for postgres conneoctor +connector.class=io.debezium.connector.mongodb.MongoDbConnector +# default snapshot mode to initial +snapshot.mode=${debezium.snapshot.mode:-initial} +mongodb.connection.string=${mongodb.url} +collection.include.list=${collection.name} +# default heartbeat interval 5 mins +heartbeat.interval.ms=${debezium.heartbeat.interval.ms:-300000} +# TODO: set this field in the code +name=${collection.name} +provide.transaction.metadata=${transactional:-false} +# update event messages include the full document +capture.mode=${debezium.capture.mode:-change_streams_update_full} +# disable tombstones event +tombstones.on.delete=${debezium.tombstones.on.delete:-false} diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties b/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties index 4c74b48bbf174..a361dcf71cefe 100644 --- a/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties +++ b/java/connector-node/risingwave-connector-service/src/main/resources/mysql.properties @@ -22,4 +22,5 @@ heartbeat.interval.ms=${debezium.heartbeat.interval.ms:-60000} # In sharing cdc mode, we will subscribe to multiple tables in the given database, # so here we set ${table.name} to a default value `RW_CDC_Sharing` just for display. name=${hostname}:${port}:${database.name}.${table.name:-RW_CDC_Sharing} +# In sharing cdc mode, transaction metadata will be enabled in frontend provide.transaction.metadata=${transactional:-false} diff --git a/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties b/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties index 7f9785b7a34b1..326138403d3b2 100644 --- a/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties +++ b/java/connector-node/risingwave-connector-service/src/main/resources/postgres.properties @@ -20,4 +20,5 @@ heartbeat.interval.ms=${debezium.heartbeat.interval.ms:-300000} # In sharing cdc source mode, we will subscribe to multiple tables in the given database, # so here we set ${table.name} to a default value `RW_CDC_Sharing` just for display. name=${hostname}:${port}:${database.name}.${schema.name}.${table.name:-RW_CDC_Sharing} +# In sharing cdc mode, transaction metadata will be enabled in frontend provide.transaction.metadata=${transactional:-false} diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkFactoryTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkFactoryTest.java deleted file mode 100755 index 86c70b80e049a..0000000000000 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkFactoryTest.java +++ /dev/null @@ -1,150 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector.sink.iceberg; - -import static org.junit.Assert.*; - -import com.google.common.collect.Lists; -import com.risingwave.connector.AppendOnlyIcebergSinkWriter; -import com.risingwave.connector.IcebergSinkFactory; -import com.risingwave.connector.TestUtils; -import com.risingwave.connector.api.TableSchema; -import com.risingwave.proto.Catalog.SinkType; -import com.risingwave.proto.Data; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.types.Types; -import org.junit.Test; - -public class IcebergSinkFactoryTest { - static String warehousePath = "file:///tmp/rw-sinknode/iceberg-sink/warehouse"; - static String databaseName = "demo_db"; - static String tableName = "demo_table"; - static String sinkMode = "append-only"; - static Schema icebergTableSchema = - new Schema( - Types.NestedField.required(1, "id", Types.IntegerType.get()), - Types.NestedField.required(2, "name", Types.StringType.get())); - - private void createMockTable() throws IOException { - if (!Paths.get(warehousePath).toFile().isDirectory()) { - Files.createDirectories(Paths.get(warehousePath)); - } - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - try { - catalog.dropTable(tableIdent); - } catch (Exception e) { - // Ignored. - } - PartitionSpec spec = PartitionSpec.unpartitioned(); - catalog.createTable(tableIdent, icebergTableSchema, spec, Map.of("format-version", "2")); - catalog.close(); - } - - @Test - public void testCreate() throws IOException { - createMockTable(); - IcebergSinkFactory sinkFactory = new IcebergSinkFactory(); - AppendOnlyIcebergSinkWriter sink = - (AppendOnlyIcebergSinkWriter) - sinkFactory.createWriter( - TestUtils.getMockTableSchema(), - Map.of( - "type", - sinkMode, - "warehouse.path", - warehousePath, - "database.name", - databaseName, - "table.name", - tableName)); - try { - assertTrue( - sink.getHadoopCatalog() - .tableExists(TableIdentifier.of(databaseName, tableName))); - assertEquals( - sink.getIcebergTable().location(), - warehousePath + "/" + databaseName + "/" + tableName); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test(expected = RuntimeException.class) - public void testValidateSchemaName() throws IOException { - createMockTable(); - IcebergSinkFactory sinkFactory = new IcebergSinkFactory(); - Map tableProperties = - Map.of( - "type", - sinkMode, - "warehouse.path", - warehousePath, - "database.name", - databaseName, - "table.name", - tableName); - TableSchema diffTypeTableSchema = - new TableSchema( - Lists.newArrayList("id", "names"), - Lists.newArrayList( - Data.DataType.newBuilder() - .setTypeName(Data.DataType.TypeName.INT32) - .build(), - Data.DataType.newBuilder() - .setTypeName(Data.DataType.TypeName.VARCHAR) - .build()), - Lists.newArrayList("id")); - sinkFactory.validate(diffTypeTableSchema, tableProperties, SinkType.SINK_TYPE_APPEND_ONLY); - } - - @Test(expected = RuntimeException.class) - public void testValidateSchemaType() throws IOException { - createMockTable(); - IcebergSinkFactory sinkFactory = new IcebergSinkFactory(); - Map tableProperties = - Map.of( - "type", - sinkMode, - "warehouse.path", - warehousePath, - "database.name", - databaseName, - "table.name", - tableName); - TableSchema diffTypeTableSchema = - new TableSchema( - Lists.newArrayList("id", "name"), - Lists.newArrayList( - Data.DataType.newBuilder() - .setTypeName(Data.DataType.TypeName.INT32) - .build(), - Data.DataType.newBuilder() - .setTypeName(Data.DataType.TypeName.INT32) - .build()), - Lists.newArrayList("id")); - sinkFactory.validate(diffTypeTableSchema, tableProperties, SinkType.SINK_TYPE_APPEND_ONLY); - } -} diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkLocalTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkLocalTest.java deleted file mode 100644 index 231148a14d7b7..0000000000000 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkLocalTest.java +++ /dev/null @@ -1,215 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector.sink.iceberg; - -import static com.risingwave.proto.Data.*; -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; - -import com.google.common.collect.Sets; -import com.risingwave.connector.AppendOnlyIcebergSinkWriter; -import com.risingwave.connector.IcebergSinkCoordinator; -import com.risingwave.connector.TestUtils; -import com.risingwave.connector.api.sink.ArraySinkRow; -import com.risingwave.proto.ConnectorServiceProto; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.*; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.IcebergGenerics; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.types.Types; -import org.apache.spark.SparkConf; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.junit.Test; - -public class IcebergSinkLocalTest { - static String warehousePath = "/tmp/rw-sinknode/iceberg-sink/warehouse"; - static String databaseName = "demo_db"; - static String tableName = "demo_table"; - static Schema icebergTableSchema = - new Schema( - Types.NestedField.required(1, "id", Types.IntegerType.get()), - Types.NestedField.required(2, "name", Types.StringType.get())); - - private void createMockTable() throws IOException { - if (!Paths.get(warehousePath).toFile().isDirectory()) { - Files.createDirectories(Paths.get(warehousePath)); - } - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - try { - catalog.dropTable(tableIdent); - } catch (Exception e) { - // Ignored. - } - PartitionSpec spec = PartitionSpec.unpartitioned(); - catalog.createTable(tableIdent, icebergTableSchema, spec, Map.of("format-version", "2")); - catalog.close(); - } - - private void validateTableWithIceberg(Set expected) { - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - Table icebergTable = catalog.loadTable(tableIdent); - CloseableIterable iter = IcebergGenerics.read(icebergTable).build(); - Set actual = Sets.newHashSet(iter); - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - private void validateTableWithSpark(Set expected) { - SparkConf sparkConf = new SparkConf(); - sparkConf.set("spark.sql.catalog.demo", "org.apache.iceberg.spark.SparkCatalog"); - sparkConf.set("spark.sql.catalog.demo.type", "hadoop"); - sparkConf.set("spark.sql.catalog.demo.warehouse", warehousePath); - sparkConf.set("spark.sql.catalog.defaultCatalog", "demo"); - SparkSession spark = SparkSession.builder().master("local").config(sparkConf).getOrCreate(); - List rows = - spark.read() - .format("iceberg") - .load(String.format("demo.%s.%s", databaseName, tableName)) - .collectAsList(); - spark.close(); - Set actual = new HashSet<>(); - for (Row row : rows) { - int id = row.getInt(0); - String name = row.getString(1); - Record record = GenericRecord.create(icebergTableSchema); - record.setField("id", id); - record.setField("name", name); - actual.add(record); - } - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - @Test - public void testSync() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - AppendOnlyIcebergSinkWriter sink = - new AppendOnlyIcebergSinkWriter( - TestUtils.getMockTableSchema(), - hadoopCatalog, - hadoopCatalog.loadTable(tableIdentifier), - FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write(Arrays.asList(new ArraySinkRow(Op.INSERT, 1, "Alice"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Alice"); - Set expected = Sets.newHashSet(record1); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - sink.beginEpoch(234); - sink.write(Arrays.asList(new ArraySinkRow(Op.INSERT, 2, "Bob"))); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - metadata = sink.barrier(true).get(); - coordinator.commit(234, Collections.singletonList(metadata)); - - Record record2 = GenericRecord.create(icebergTableSchema); - record2.setField("id", 2); - record2.setField("name", "Bob"); - expected.add(record2); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testWrite() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - AppendOnlyIcebergSinkWriter sink = - new AppendOnlyIcebergSinkWriter( - TestUtils.getMockTableSchema(), - hadoopCatalog, - icebergTable, - FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write( - Arrays.asList( - new ArraySinkRow(Op.INSERT, 1, "Alice"), - new ArraySinkRow(Op.INSERT, 2, "Bob"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Alice"); - Record record2 = GenericRecord.create(icebergTableSchema); - record2.setField("id", 2); - record2.setField("name", "Bob"); - Set expected = Sets.newHashSet(record1, record2); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testDrop() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - AppendOnlyIcebergSinkWriter sink = - new AppendOnlyIcebergSinkWriter( - TestUtils.getMockTableSchema(), - hadoopCatalog, - icebergTable, - FileFormat.PARQUET); - - sink.drop(); - - assertTrue(sink.isClosed()); - assertTrue(Files.exists(Paths.get(warehousePath))); - } -} diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkPartitionTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkPartitionTest.java deleted file mode 100644 index 5148bb640c915..0000000000000 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/IcebergSinkPartitionTest.java +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector.sink.iceberg; - -import static com.risingwave.proto.Data.*; -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.risingwave.connector.AppendOnlyIcebergSinkWriter; -import com.risingwave.connector.IcebergSinkCoordinator; -import com.risingwave.connector.api.TableSchema; -import com.risingwave.connector.api.sink.ArraySinkRow; -import com.risingwave.proto.ConnectorServiceProto; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.IcebergGenerics; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.types.Types; -import org.apache.spark.SparkConf; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.junit.Test; - -public class IcebergSinkPartitionTest { - static String warehousePath = "/tmp/rw-sinknode/iceberg-sink/warehouse"; - static String databaseName = "demo_db"; - static String tableName = "demo_table_partitioned"; - static Schema icebergTableSchema = - new Schema( - Types.NestedField.required(1, "id", Types.IntegerType.get()), - Types.NestedField.required(2, "name", Types.StringType.get()), - Types.NestedField.required(3, "part", Types.StringType.get())); - static TableSchema tableSchema = - new TableSchema( - Lists.newArrayList("id", "name", "part"), - Lists.newArrayList( - DataType.newBuilder().setTypeName(DataType.TypeName.INT32).build(), - DataType.newBuilder().setTypeName(DataType.TypeName.VARCHAR).build(), - DataType.newBuilder().setTypeName(DataType.TypeName.VARCHAR).build()), - Lists.newArrayList("id")); - - private void createMockTable() throws IOException { - if (!Paths.get(warehousePath).toFile().isDirectory()) { - Files.createDirectories(Paths.get(warehousePath)); - } - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - try { - catalog.dropTable(tableIdent); - } catch (Exception e) { - // Ignored. - } - PartitionSpec spec = PartitionSpec.builderFor(icebergTableSchema).identity("part").build(); - catalog.createTable(tableIdent, icebergTableSchema, spec, Map.of("format-version", "2")); - catalog.close(); - } - - private void validateTableWithIceberg(Set expected) { - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - Table icebergTable = catalog.loadTable(tableIdent); - CloseableIterable iter = IcebergGenerics.read(icebergTable).build(); - Set actual = Sets.newHashSet(iter); - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - private void validateTableWithSpark(Set expected) { - SparkConf sparkConf = new SparkConf(); - sparkConf.set("spark.sql.catalog.demo", "org.apache.iceberg.spark.SparkCatalog"); - sparkConf.set("spark.sql.catalog.demo.type", "hadoop"); - sparkConf.set("spark.sql.catalog.demo.warehouse", warehousePath); - sparkConf.set("spark.sql.catalog.defaultCatalog", "demo"); - SparkSession spark = SparkSession.builder().master("local").config(sparkConf).getOrCreate(); - List rows = - spark.read() - .format("iceberg") - .load(String.format("demo.%s.%s", databaseName, tableName)) - .collectAsList(); - spark.close(); - Set actual = new HashSet<>(); - for (Row row : rows) { - int id = row.getInt(0); - String name = row.getString(1); - String part = row.getString(2); - Record record = GenericRecord.create(icebergTableSchema); - record.setField("id", id); - record.setField("name", name); - record.setField("part", part); - actual.add(record); - } - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - @Test - public void testSync() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - AppendOnlyIcebergSinkWriter sink = - new AppendOnlyIcebergSinkWriter( - tableSchema, hadoopCatalog, icebergTable, FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write(Arrays.asList(new ArraySinkRow(Op.INSERT, 1, "Alice", "aaa"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Alice"); - record1.setField("part", "aaa"); - Set expected = Sets.newHashSet(record1); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - sink.beginEpoch(234); - sink.write(Arrays.asList(new ArraySinkRow(Op.INSERT, 2, "Bob", "bbb"))); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - metadata = sink.barrier(true).get(); - coordinator.commit(234, Collections.singletonList(metadata)); - - Record record2 = GenericRecord.create(icebergTableSchema); - record2.setField("id", 2); - record2.setField("name", "Bob"); - record2.setField("part", "bbb"); - expected.add(record2); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testWrite() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - AppendOnlyIcebergSinkWriter sink = - new AppendOnlyIcebergSinkWriter( - tableSchema, hadoopCatalog, icebergTable, FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write( - Arrays.asList( - new ArraySinkRow(Op.INSERT, 1, "Alice", "aaa"), - new ArraySinkRow(Op.INSERT, 2, "Bob", "bbb"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Alice"); - record1.setField("part", "aaa"); - Record record2 = GenericRecord.create(icebergTableSchema); - record2.setField("id", 2); - record2.setField("name", "Bob"); - record2.setField("part", "bbb"); - Set expected = Sets.newHashSet(record1, record2); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testDrop() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - AppendOnlyIcebergSinkWriter sink = - new AppendOnlyIcebergSinkWriter( - tableSchema, - hadoopCatalog, - hadoopCatalog.loadTable(tableIdentifier), - FileFormat.PARQUET); - - sink.drop(); - - assertTrue(sink.isClosed()); - assertTrue(Files.exists(Paths.get(warehousePath))); - } -} diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/SinkRowMapTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/SinkRowMapTest.java deleted file mode 100644 index 60caee0ee4c35..0000000000000 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/SinkRowMapTest.java +++ /dev/null @@ -1,209 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector.sink.iceberg; - -import static org.junit.Assert.assertEquals; - -import com.risingwave.connector.SinkRowMap; -import com.risingwave.connector.api.sink.ArraySinkRow; -import com.risingwave.connector.api.sink.SinkRow; -import com.risingwave.proto.Data; -import java.util.ArrayList; -import java.util.List; -import org.apache.iceberg.Schema; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.types.Types; -import org.junit.Assert; -import org.junit.Test; - -public class SinkRowMapTest { - @Test - public void testInsert() { - SinkRowMap sinkRowMap = new SinkRowMap(); - SinkRow row = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1); - List> key = new ArrayList<>(); - key.add((Comparable) row.get(0)); - Schema schema = new Schema(Types.NestedField.optional(0, "id", Types.IntegerType.get())); - Record r = GenericRecord.create(schema); - r.set(0, row.get(0)); - - sinkRowMap.insert(key, r); - assertEquals(1, sinkRowMap.getMap().size()); - assertEquals(null, sinkRowMap.getMap().get(key).getDelete()); - assertEquals(r, sinkRowMap.getMap().get(key).getInsert()); - } - - @Test - public void testInsertAfterDelete() { - SinkRowMap sinkRowMap = new SinkRowMap(); - Schema schema = - new Schema( - Types.NestedField.optional(0, "id", Types.IntegerType.get()), - Types.NestedField.optional(1, "name", Types.StringType.get())); - - SinkRow row1 = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1, "Alice"); - List> key1 = new ArrayList<>(); - key1.add((Comparable) row1.get(0)); - Record r1 = GenericRecord.create(schema); - r1.set(0, row1.get(0)); - r1.set(1, row1.get(1)); - SinkRow row2 = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1, "Bob"); - List> key2 = new ArrayList<>(); - key2.add((Comparable) row2.get(0)); - Record r2 = GenericRecord.create(schema); - r2.set(0, row2.get(0)); - r2.set(1, row2.get(1)); - - sinkRowMap.delete(key1, r1); - sinkRowMap.insert(key1, r2); - assertEquals(1, sinkRowMap.getMap().size()); - assertEquals(r1, sinkRowMap.getMap().get(key1).getDelete()); - assertEquals(r2, sinkRowMap.getMap().get(key1).getInsert()); - } - - @Test - public void testInsertAfterInsert() { - SinkRowMap sinkRowMap = new SinkRowMap(); - SinkRow row = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1); - List> key = new ArrayList<>(); - key.add((Comparable) row.get(0)); - Schema schema = new Schema(Types.NestedField.optional(0, "id", Types.IntegerType.get())); - Record r = GenericRecord.create(schema); - r.set(0, row.get(0)); - - sinkRowMap.insert(key, r); - boolean exceptionThrown = false; - try { - sinkRowMap.insert(key, r); - } catch (RuntimeException e) { - exceptionThrown = true; - Assert.assertTrue( - e.getMessage() - .toLowerCase() - .contains("try to insert a duplicated primary key")); - } - if (!exceptionThrown) { - Assert.fail("Expected exception not thrown: `try to insert a duplicated primary key`"); - } - } - - @Test - public void testDelete() { - SinkRowMap sinkRowMap = new SinkRowMap(); - - SinkRow row = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1); - List> key = new ArrayList<>(); - key.add((Comparable) row.get(0)); - - Schema schema = new Schema(Types.NestedField.optional(0, "id", Types.IntegerType.get())); - Record r = GenericRecord.create(schema); - r.set(0, row.get(0)); - - sinkRowMap.delete(key, r); - assertEquals(1, sinkRowMap.getMap().size()); - assertEquals(null, sinkRowMap.getMap().get(key).getInsert()); - assertEquals(r, sinkRowMap.getMap().get(key).getDelete()); - } - - @Test - public void testDeleteAfterDelete() { - SinkRowMap sinkRowMap = new SinkRowMap(); - SinkRow row = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1); - List> key = new ArrayList<>(); - key.add((Comparable) row.get(0)); - - Schema schema = new Schema(Types.NestedField.optional(0, "id", Types.IntegerType.get())); - Record r = GenericRecord.create(schema); - r.set(0, row.get(0)); - - sinkRowMap.delete(key, r); - boolean exceptionThrown = false; - try { - sinkRowMap.delete(key, r); - } catch (RuntimeException e) { - exceptionThrown = true; - Assert.assertTrue( - e.getMessage().toLowerCase().contains("try to double delete a primary key")); - } - if (!exceptionThrown) { - Assert.fail("Expected exception not thrown: `try to double delete a primary key`"); - } - } - - @Test - public void testDeleteAfterInsert() { - SinkRowMap sinkRowMap = new SinkRowMap(); - - SinkRow row = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1); - List> key = new ArrayList<>(); - key.add((Comparable) row.get(0)); - - Schema schema = new Schema(Types.NestedField.optional(0, "id", Types.IntegerType.get())); - Record r = GenericRecord.create(schema); - r.set(0, row.get(0)); - - sinkRowMap.insert(key, r); - sinkRowMap.delete(key, r); - assertEquals(0, sinkRowMap.getMap().size()); - } - - @Test - public void testDeleteAfterUpdate() { - SinkRowMap sinkRowMap = new SinkRowMap(); - - Schema schema = - new Schema( - Types.NestedField.optional(0, "id", Types.IntegerType.get()), - Types.NestedField.optional(1, "name", Types.StringType.get())); - - SinkRow row1 = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1, "Alice"); - List> key1 = new ArrayList<>(); - key1.add((Comparable) row1.get(0)); - Record r1 = GenericRecord.create(schema); - r1.set(0, row1.get(0)); - r1.set(1, row1.get(1)); - - SinkRow row2 = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1, "Clare"); - List> key2 = new ArrayList<>(); - key2.add((Comparable) row2.get(0)); - Record r2 = GenericRecord.create(schema); - r2.set(0, row2.get(0)); - r2.set(1, row2.get(1)); - - sinkRowMap.delete(key1, r1); - sinkRowMap.insert(key2, r2); - sinkRowMap.delete(key2, r2); - assertEquals(1, sinkRowMap.getMap().size()); - assertEquals(null, sinkRowMap.getMap().get(key1).getInsert()); - assertEquals(r1, sinkRowMap.getMap().get(key1).getDelete()); - } - - @Test - public void testClear() { - SinkRowMap sinkRowMap = new SinkRowMap(); - - SinkRow row = new ArraySinkRow(Data.Op.OP_UNSPECIFIED, 1); - List> key = new ArrayList<>(); - key.add((Comparable) row.get(0)); - Schema schema = new Schema(Types.NestedField.optional(0, "id", Types.IntegerType.get())); - Record r = GenericRecord.create(schema); - r.set(0, row.get(0)); - sinkRowMap.insert(key, r); - - sinkRowMap.clear(); - assertEquals(0, sinkRowMap.getMap().size()); - } -} diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/UpsertIcebergSinkLocalTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/UpsertIcebergSinkLocalTest.java deleted file mode 100644 index 1047e1bff51dc..0000000000000 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/UpsertIcebergSinkLocalTest.java +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector; - -import static com.risingwave.proto.Data.*; -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; - -import com.google.common.collect.Sets; -import com.risingwave.connector.api.sink.ArraySinkRow; -import com.risingwave.proto.ConnectorServiceProto; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.*; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.IcebergGenerics; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.types.Types; -import org.apache.spark.SparkConf; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.junit.Test; - -public class UpsertIcebergSinkLocalTest { - static String warehousePath = "/tmp/rw-sinknode/iceberg-sink/warehouse"; - static String databaseName = "demo_db"; - static String tableName = "demo_table"; - static Schema icebergTableSchema = - new Schema( - Types.NestedField.required(1, "id", Types.IntegerType.get()), - Types.NestedField.required(2, "name", Types.StringType.get())); - - private void createMockTable() throws IOException { - if (!Paths.get(warehousePath).toFile().isDirectory()) { - Files.createDirectories(Paths.get(warehousePath)); - } - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - try { - catalog.dropTable(tableIdent); - } catch (Exception e) { - // Ignored. - } - PartitionSpec spec = PartitionSpec.unpartitioned(); - catalog.createTable(tableIdent, icebergTableSchema, spec, Map.of("format-version", "2")); - catalog.close(); - } - - private void validateTableWithIceberg(Set expected) { - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - Table icebergTable = catalog.loadTable(tableIdent); - CloseableIterable iter = IcebergGenerics.read(icebergTable).build(); - Set actual = Sets.newHashSet(iter); - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - private void validateTableWithSpark(Set expected) { - SparkConf sparkConf = new SparkConf(); - sparkConf.set("spark.sql.catalog.demo", "org.apache.iceberg.spark.SparkCatalog"); - sparkConf.set("spark.sql.catalog.demo.type", "hadoop"); - sparkConf.set("spark.sql.catalog.demo.warehouse", warehousePath); - sparkConf.set("spark.sql.catalog.defaultCatalog", "demo"); - SparkSession spark = SparkSession.builder().master("local").config(sparkConf).getOrCreate(); - List rows = - spark.read() - .format("iceberg") - .load(String.format("demo.%s.%s", databaseName, tableName)) - .collectAsList(); - spark.close(); - Set actual = new HashSet<>(); - for (Row row : rows) { - int id = row.getInt(0); - String name = row.getString(1); - Record record = GenericRecord.create(icebergTableSchema); - record.setField("id", id); - record.setField("name", name); - actual.add(record); - } - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - @Test - public void testSync() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - UpsertIcebergSinkWriter sink = - new UpsertIcebergSinkWriter( - TestUtils.getMockTableSchema(), - hadoopCatalog, - icebergTable, - FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write(Arrays.asList(new ArraySinkRow(Op.INSERT, 1, "Alice"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Alice"); - Set expected = Sets.newHashSet(record1); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - sink.beginEpoch(234); - sink.write(Arrays.asList(new ArraySinkRow(Op.INSERT, 2, "Bob"))); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - metadata = sink.barrier(true).get(); - coordinator.commit(234, Collections.singletonList(metadata)); - - Record record2 = GenericRecord.create(icebergTableSchema); - record2.setField("id", 2); - record2.setField("name", "Bob"); - expected.add(record2); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testWrite() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - UpsertIcebergSinkWriter sink = - new UpsertIcebergSinkWriter( - TestUtils.getMockTableSchema(), - hadoopCatalog, - icebergTable, - FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write( - Arrays.asList( - new ArraySinkRow(Op.INSERT, 1, "Alice"), - new ArraySinkRow(Op.INSERT, 2, "Bob"), - new ArraySinkRow(Op.UPDATE_DELETE, 1, "Alice"), - new ArraySinkRow(Op.UPDATE_INSERT, 1, "Clare"), - new ArraySinkRow(Op.DELETE, 2, "Bob"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Clare"); - Set expected = Sets.newHashSet(record1); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - sink.beginEpoch(234); - sink.write( - Arrays.asList( - new ArraySinkRow(Op.UPDATE_DELETE, 1, "Clare"), - new ArraySinkRow(Op.UPDATE_INSERT, 1, "Alice"), - new ArraySinkRow(Op.DELETE, 1, "Alice"))); - metadata = sink.barrier(true).get(); - coordinator.commit(234, Collections.singletonList(metadata)); - - validateTableWithIceberg(Sets.newHashSet()); - validateTableWithSpark(Sets.newHashSet()); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testDrop() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - UpsertIcebergSinkWriter sink = - new UpsertIcebergSinkWriter( - TestUtils.getMockTableSchema(), - hadoopCatalog, - hadoopCatalog.loadTable(tableIdentifier), - FileFormat.PARQUET); - - sink.drop(); - - assertTrue(sink.isClosed()); - assertTrue(Files.exists(Paths.get(warehousePath))); - } -} diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/UpsertIcebergSinkPartitionTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/UpsertIcebergSinkPartitionTest.java deleted file mode 100644 index f41f352ef8df9..0000000000000 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/iceberg/UpsertIcebergSinkPartitionTest.java +++ /dev/null @@ -1,235 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector.sink.iceberg; - -import static com.risingwave.proto.Data.*; -import static org.junit.Assert.*; -import static org.junit.Assert.assertEquals; - -import com.google.common.collect.Lists; -import com.google.common.collect.Sets; -import com.risingwave.connector.IcebergSinkCoordinator; -import com.risingwave.connector.UpsertIcebergSinkWriter; -import com.risingwave.connector.api.TableSchema; -import com.risingwave.connector.api.sink.ArraySinkRow; -import com.risingwave.proto.ConnectorServiceProto; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.util.*; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.*; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.IcebergGenerics; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.io.CloseableIterable; -import org.apache.iceberg.types.Types; -import org.apache.spark.SparkConf; -import org.apache.spark.sql.Row; -import org.apache.spark.sql.SparkSession; -import org.junit.Test; - -public class UpsertIcebergSinkPartitionTest { - static String warehousePath = "/tmp/rw-sinknode/iceberg-sink/warehouse"; - static String databaseName = "demo_db"; - static String tableName = "demo_table_partitioned"; - static Schema icebergTableSchema = - new Schema( - Types.NestedField.required(1, "id", Types.IntegerType.get()), - Types.NestedField.required(2, "name", Types.StringType.get()), - Types.NestedField.required(3, "part", Types.StringType.get())); - - static TableSchema tableSchema = - new TableSchema( - Lists.newArrayList("id", "name", "part"), - Lists.newArrayList( - DataType.newBuilder().setTypeName(DataType.TypeName.INT32).build(), - DataType.newBuilder().setTypeName(DataType.TypeName.VARCHAR).build(), - DataType.newBuilder().setTypeName(DataType.TypeName.VARCHAR).build()), - Lists.newArrayList("id")); - - private void createMockTable() throws IOException { - if (!Paths.get(warehousePath).toFile().isDirectory()) { - Files.createDirectories(Paths.get(warehousePath)); - } - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - try { - catalog.dropTable(tableIdent); - } catch (Exception e) { - // Ignored. - } - PartitionSpec spec = PartitionSpec.builderFor(icebergTableSchema).identity("part").build(); - catalog.createTable(tableIdent, icebergTableSchema, spec, Map.of("format-version", "2")); - catalog.close(); - } - - private void validateTableWithIceberg(Set expected) { - HadoopCatalog catalog = new HadoopCatalog(new Configuration(), warehousePath); - TableIdentifier tableIdent = TableIdentifier.of(databaseName, tableName); - Table icebergTable = catalog.loadTable(tableIdent); - CloseableIterable iter = IcebergGenerics.read(icebergTable).build(); - Set actual = Sets.newHashSet(iter); - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - private void validateTableWithSpark(Set expected) { - SparkConf sparkConf = new SparkConf(); - sparkConf.set("spark.sql.catalog.demo", "org.apache.iceberg.spark.SparkCatalog"); - sparkConf.set("spark.sql.catalog.demo.type", "hadoop"); - sparkConf.set("spark.sql.catalog.demo.warehouse", warehousePath); - sparkConf.set("spark.sql.catalog.defaultCatalog", "demo"); - SparkSession spark = SparkSession.builder().master("local").config(sparkConf).getOrCreate(); - List rows = - spark.read() - .format("iceberg") - .load(String.format("demo.%s.%s", databaseName, tableName)) - .collectAsList(); - spark.close(); - Set actual = new HashSet<>(); - for (Row row : rows) { - int id = row.getInt(0); - String name = row.getString(1); - String part = row.getString(2); - Record record = GenericRecord.create(icebergTableSchema); - record.setField("id", id); - record.setField("name", name); - record.setField("part", part); - actual.add(record); - } - assertEquals(expected.size(), actual.size()); - assertEquals(expected, actual); - } - - @Test - public void testSync() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - UpsertIcebergSinkWriter sink = - new UpsertIcebergSinkWriter( - tableSchema, hadoopCatalog, icebergTable, FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write(Collections.singletonList(new ArraySinkRow(Op.INSERT, 1, "Alice", "aaa"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Alice"); - record1.setField("part", "aaa"); - Set expected = Sets.newHashSet(record1); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - sink.beginEpoch(234); - sink.write(Collections.singletonList((new ArraySinkRow(Op.INSERT, 2, "Bob", "bbb")))); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - metadata = sink.barrier(true).get(); - coordinator.commit(234, Collections.singletonList(metadata)); - - Record record2 = GenericRecord.create(icebergTableSchema); - record2.setField("id", 2); - record2.setField("name", "Bob"); - record2.setField("part", "bbb"); - expected.add(record2); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testWrite() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkCoordinator coordinator = new IcebergSinkCoordinator(icebergTable); - UpsertIcebergSinkWriter sink = - new UpsertIcebergSinkWriter( - tableSchema, hadoopCatalog, icebergTable, FileFormat.PARQUET); - - try { - sink.beginEpoch(233); - sink.write( - Arrays.asList( - new ArraySinkRow(Op.INSERT, 1, "Alice", "aaa"), - new ArraySinkRow(Op.INSERT, 2, "Bob", "bbb"), - new ArraySinkRow(Op.UPDATE_DELETE, 1, "Alice", "aaa"), - new ArraySinkRow(Op.UPDATE_INSERT, 1, "Clare", "ccc"), - new ArraySinkRow(Op.DELETE, 2, "Bob", "bbb"))); - ConnectorServiceProto.SinkMetadata metadata = sink.barrier(true).get(); - coordinator.commit(233, Collections.singletonList(metadata)); - - Record record1 = GenericRecord.create(icebergTableSchema); - record1.setField("id", 1); - record1.setField("name", "Clare"); - record1.setField("part", "ccc"); - Set expected = Sets.newHashSet(record1); - validateTableWithIceberg(expected); - validateTableWithSpark(expected); - - sink.beginEpoch(234); - sink.write( - Arrays.asList( - new ArraySinkRow(Op.UPDATE_DELETE, 1, "Clare", "ccc"), - new ArraySinkRow(Op.UPDATE_INSERT, 1, "Alice", "aaa"), - new ArraySinkRow(Op.DELETE, 1, "Alice", "aaa"))); - metadata = sink.barrier(true).get(); - coordinator.commit(234, Collections.singletonList(metadata)); - - validateTableWithIceberg(Sets.newHashSet()); - validateTableWithSpark(Sets.newHashSet()); - } catch (Exception e) { - fail("Exception: " + e); - } finally { - sink.drop(); - } - } - - @Test - public void testDrop() throws IOException { - createMockTable(); - Configuration hadoopConf = new Configuration(); - HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); - TableIdentifier tableIdentifier = TableIdentifier.of(databaseName, tableName); - UpsertIcebergSinkWriter sink = - new UpsertIcebergSinkWriter( - tableSchema, - hadoopCatalog, - hadoopCatalog.loadTable(tableIdentifier), - FileFormat.PARQUET); - - sink.drop(); - - assertTrue(sink.isClosed()); - assertTrue(Files.exists(Paths.get(warehousePath))); - } -} diff --git a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/jdbc/JDBCSinkTest.java b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/jdbc/JDBCSinkTest.java index 68e9f386b2152..12ac3ea8b7f86 100644 --- a/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/jdbc/JDBCSinkTest.java +++ b/java/connector-node/risingwave-connector-test/src/test/java/com/risingwave/connector/sink/jdbc/JDBCSinkTest.java @@ -234,7 +234,7 @@ static void testJDBCWrite(JdbcDatabaseContainer container, TestType testType) rs.getObject(5, LocalDateTime.class)); assertEquals( OffsetDateTime.of(1970, 1, 1, 0, 0, 1, 1000, ZoneOffset.UTC), - rs.getObject(6, OffsetDateTime.class)); + rs.getObject(6, OffsetDateTime.class).toInstant().atOffset(ZoneOffset.UTC)); assertEquals( "{\"key\": \"password\", \"value\": \"Singularity123123123123\"}", rs.getString(7)); @@ -282,7 +282,7 @@ public void testPostgres() throws SQLException { @Test public void testMySQL() throws SQLException { MySQLContainer mysql = - new MySQLContainer<>("mysql:8") + new MySQLContainer<>("mysql:8.0") .withDatabaseName("test") .withUsername("postgres") .withPassword("password") diff --git a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraConfig.java b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraConfig.java index 9ac3d257b2bad..7c883335cfc23 100644 --- a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraConfig.java +++ b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraConfig.java @@ -23,6 +23,7 @@ public class CassandraConfig extends CommonSinkConfig { /** Required */ private String type; + /** Required */ private String url; diff --git a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraSink.java b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraSink.java index d50010d6e2635..b0b7fb93c7b51 100644 --- a/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraSink.java +++ b/java/connector-node/risingwave-sink-cassandra/src/main/java/com/risingwave/connector/CassandraSink.java @@ -34,6 +34,8 @@ public class CassandraSink extends SinkWriterBase { private static final Logger LOG = LoggerFactory.getLogger(CassandraSink.class); + private static final Integer MAX_BATCH_SIZE = 1024 * 16; + private final CqlSession session; private final List updateRowCache = new ArrayList<>(1); private final HashMap stmtMap; @@ -122,6 +124,7 @@ private void write_append_only(Iterator rows) { .withDescription("Unknown operation: " + op) .asRuntimeException(); } + tryCommit(); } } @@ -155,6 +158,13 @@ private void write_upsert(Iterator rows) { .withDescription("Unknown operation: " + op) .asRuntimeException(); } + tryCommit(); + } + } + + private void tryCommit() { + if (batchBuilder.getStatementsCount() >= MAX_BATCH_SIZE) { + sync(); } } diff --git a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink.java b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink.java index 1e906ba409556..e40332a327112 100644 --- a/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink.java +++ b/java/connector-node/risingwave-sink-es-7/src/main/java/com/risingwave/connector/EsSink.java @@ -248,7 +248,7 @@ public BulkListener(RequestTracker requestTracker) { /** This method is called just before bulk is executed. */ @Override public void beforeBulk(long executionId, BulkRequest request) { - LOG.info("Sending bulk of {} actions to Elasticsearch.", request.numberOfActions()); + LOG.debug("Sending bulk of {} actions to Elasticsearch.", request.numberOfActions()); } /** This method is called after bulk execution. */ @@ -262,7 +262,7 @@ public void afterBulk(long executionId, BulkRequest request, BulkResponse respon this.requestTracker.addErrResult(errMessage); } else { this.requestTracker.addOkResult(request.numberOfActions()); - LOG.info("Sent bulk of {} actions to Elasticsearch.", request.numberOfActions()); + LOG.debug("Sent bulk of {} actions to Elasticsearch.", request.numberOfActions()); } } diff --git a/java/connector-node/risingwave-sink-iceberg/pom.xml b/java/connector-node/risingwave-sink-iceberg/pom.xml index fdd2b330a972c..9f733d830a475 100644 --- a/java/connector-node/risingwave-sink-iceberg/pom.xml +++ b/java/connector-node/risingwave-sink-iceberg/pom.xml @@ -16,7 +16,7 @@ risingwave-sink-iceberg - 1.0.0 + 1.4.1 11 11 true @@ -45,6 +45,16 @@ iceberg-core ${iceberg.version} + + org.apache.iceberg + iceberg-api + ${iceberg.version} + + + org.apache.iceberg + iceberg-bundled-guava + ${iceberg.version} + org.apache.iceberg iceberg-parquet @@ -55,11 +65,36 @@ iceberg-data ${iceberg.version} + + org.apache.iceberg + iceberg-hive-metastore + ${iceberg.version} + + + org.apache.hive + hive-metastore + org.apache.parquet parquet-avro 1.12.3 + + org.apache.hadoop + hadoop-common + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + org.apache.iceberg iceberg-aws @@ -68,13 +103,37 @@ software.amazon.awssdk s3 - 2.18.20 software.amazon.awssdk - url-connection-client - 2.18.20 + sts + + + org.postgresql + postgresql + + + mysql + mysql-connector-java + + + org.xerial + sqlite-jdbc + + + software.amazon.awssdk + apache-client + + + junit + junit + test + + + org.assertj + assertj-core + 3.24.2 + test - diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/AppendOnlyIcebergSinkWriter.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/AppendOnlyIcebergSinkWriter.java deleted file mode 100644 index c7c454c812530..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/AppendOnlyIcebergSinkWriter.java +++ /dev/null @@ -1,146 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector; - -import static io.grpc.Status.INTERNAL; -import static io.grpc.Status.UNIMPLEMENTED; - -import com.risingwave.connector.api.TableSchema; -import com.risingwave.connector.api.sink.SinkRow; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import org.apache.iceberg.*; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.parquet.GenericParquetWriter; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.io.DataWriter; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.parquet.Parquet; - -public class AppendOnlyIcebergSinkWriter extends IcebergSinkWriterBase { - private Map> dataWriterMap = new HashMap<>(); - private boolean closed = false; - - public AppendOnlyIcebergSinkWriter( - TableSchema tableSchema, - HadoopCatalog hadoopCatalog, - Table icebergTable, - FileFormat fileFormat) { - super( - tableSchema, - icebergTable, - hadoopCatalog, - icebergTable.schema().select(Arrays.asList(tableSchema.getColumnNames())), - fileFormat); - } - - @Override - public boolean write(Iterable rows) { - for (SinkRow row : rows) { - switch (row.getOp()) { - case INSERT: - Record record = GenericRecord.create(rowSchema); - if (row.size() != tableSchema.getColumnNames().length) { - throw INTERNAL.withDescription("row values do not match table schema") - .asRuntimeException(); - } - for (int i = 0; i < rowSchema.columns().size(); i++) { - record.set(i, row.get(i)); - } - PartitionKey partitionKey = - new PartitionKey(icebergTable.spec(), icebergTable.schema()); - partitionKey.partition(record); - DataWriter dataWriter; - if (dataWriterMap.containsKey(partitionKey)) { - dataWriter = dataWriterMap.get(partitionKey); - } else { - try { - String filename = fileFormat.addExtension(UUID.randomUUID().toString()); - OutputFile outputFile = - icebergTable - .io() - .newOutputFile( - icebergTable.location() - + "/data/" - + icebergTable - .spec() - .partitionToPath(partitionKey) - + "/" - + filename); - dataWriter = - Parquet.writeData(outputFile) - .schema(rowSchema) - .withSpec(icebergTable.spec()) - .withPartition(partitionKey) - .createWriterFunc(GenericParquetWriter::buildWriter) - .overwrite() - .build(); - } catch (Exception e) { - throw INTERNAL.withDescription("failed to create dataWriter") - .asRuntimeException(); - } - dataWriterMap.put(partitionKey, dataWriter); - } - dataWriter.write(record); - break; - default: - throw UNIMPLEMENTED - .withDescription("unsupported operation: " + row.getOp()) - .asRuntimeException(); - } - } - return false; - } - - @Override - protected IcebergMetadata collectSinkMetadata() { - try { - List dataFileList = new ArrayList<>(); - for (DataWriter dataWriter : dataWriterMap.values()) { - dataWriter.close(); - DataFile dataFile = dataWriter.toDataFile(); - dataFileList.add(dataFile); - } - dataWriterMap.clear(); - return new IcebergMetadata(dataFileList.toArray(new DataFile[0]), new DeleteFile[0]); - } catch (Exception e) { - throw INTERNAL.withDescription(String.format("failed to collect metadata: %s", e)) - .withCause(e) - .asRuntimeException(); - } - } - - @Override - public void drop() { - try { - for (DataWriter dataWriter : dataWriterMap.values()) { - dataWriter.close(); - } - hadoopCatalog.close(); - closed = true; - } catch (Exception e) { - throw INTERNAL.withCause(e).asRuntimeException(); - } - } - - public boolean isClosed() { - return closed; - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergMetadata.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergMetadata.java deleted file mode 100644 index c2568a8eaf9c1..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergMetadata.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2024 RisingWave Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.risingwave.connector; - -import java.io.Serializable; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; - -public class IcebergMetadata implements Serializable { - final DataFile[] dataFiles; - final DeleteFile[] deleteFiles; - - public IcebergMetadata(DataFile[] dataFiles, DeleteFile[] deleteFiles) { - this.dataFiles = dataFiles; - this.deleteFiles = deleteFiles; - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkConfig.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkConfig.java deleted file mode 100644 index fd89fc0163497..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkConfig.java +++ /dev/null @@ -1,69 +0,0 @@ -/* - * Copyright 2024 RisingWave Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.risingwave.connector; - -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.risingwave.connector.common.S3Config; - -public class IcebergSinkConfig extends S3Config { - private String sinkType; - - private String warehousePath; - - private String databaseName; - - private String tableName; - - @JsonProperty(value = "force_append_only") - private Boolean forceAppendOnly; - - @JsonProperty(value = "primary_key") - private String primaryKey; - - @JsonCreator - public IcebergSinkConfig( - @JsonProperty(value = "type") String sinkType, - @JsonProperty(value = "warehouse.path") String warehousePath, - @JsonProperty(value = "database.name") String databaseName, - @JsonProperty(value = "table.name") String tableName) { - this.sinkType = sinkType; - this.warehousePath = warehousePath; - this.databaseName = databaseName; - this.tableName = tableName; - } - - public String getSinkType() { - return sinkType; - } - - public String getWarehousePath() { - return warehousePath; - } - - public void setWarehousePath(String warehousePath) { - this.warehousePath = warehousePath; - } - - public String getDatabaseName() { - return databaseName; - } - - public String getTableName() { - return tableName; - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkCoordinator.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkCoordinator.java deleted file mode 100644 index f4ff45cb5bfb5..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkCoordinator.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2024 RisingWave Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.risingwave.connector; - -import com.risingwave.connector.api.sink.SinkCoordinator; -import com.risingwave.java.utils.ObjectSerde; -import com.risingwave.proto.ConnectorServiceProto; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; -import org.apache.iceberg.AppendFiles; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; -import org.apache.iceberg.RowDelta; -import org.apache.iceberg.Table; -import org.apache.iceberg.Transaction; - -public class IcebergSinkCoordinator implements SinkCoordinator { - - private final Table icebergTable; - - public IcebergSinkCoordinator(Table icebergTable) { - this.icebergTable = icebergTable; - } - - @Override - public void commit(long epoch, List metadataList) { - List dataFileList = new ArrayList<>(metadataList.size()); - List deleteFileList = new ArrayList<>(metadataList.size()); - for (ConnectorServiceProto.SinkMetadata metadata : metadataList) { - IcebergMetadata icebergMetadata = - (IcebergMetadata) - ObjectSerde.deserializeObject( - metadata.getSerialized().getMetadata().toByteArray()); - dataFileList.addAll( - Arrays.stream(icebergMetadata.dataFiles).collect(Collectors.toList())); - deleteFileList.addAll( - Arrays.stream(icebergMetadata.deleteFiles).collect(Collectors.toList())); - } - boolean nonEmpty = false; - Transaction txn = icebergTable.newTransaction(); - if (!deleteFileList.isEmpty()) { - RowDelta rowDelta = txn.newRowDelta(); - for (DeleteFile deleteFile : deleteFileList) { - rowDelta.addDeletes(deleteFile); - } - rowDelta.commit(); - nonEmpty = true; - } - - if (!dataFileList.isEmpty()) { - AppendFiles append = txn.newAppend(); - for (DataFile dataFile : dataFileList) { - append.appendFile(dataFile); - } - append.commit(); - nonEmpty = true; - } - - if (nonEmpty) { - txn.commitTransaction(); - } - } - - @Override - public void drop() {} -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkFactory.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkFactory.java deleted file mode 100644 index fd8317e94e039..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkFactory.java +++ /dev/null @@ -1,187 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector; - -import static io.grpc.Status.UNIMPLEMENTED; - -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.risingwave.connector.api.TableSchema; -import com.risingwave.connector.api.sink.SinkCoordinator; -import com.risingwave.connector.api.sink.SinkFactory; -import com.risingwave.connector.api.sink.SinkWriter; -import com.risingwave.connector.common.S3Utils; -import com.risingwave.java.utils.UrlParser; -import com.risingwave.proto.Catalog.SinkType; -import io.grpc.Status; -import java.util.Map; -import org.apache.hadoop.conf.Configuration; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.Table; -import org.apache.iceberg.catalog.TableIdentifier; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class IcebergSinkFactory implements SinkFactory { - - private static final Logger LOG = LoggerFactory.getLogger(IcebergSinkFactory.class); - - public static final FileFormat FILE_FORMAT = FileFormat.PARQUET; - - // hadoop catalog config - - private static final String confIoImpl = "io-impl"; - private static final String s3FileIOImpl = "org.apache.iceberg.aws.s3.S3FileIO"; - - @Override - public SinkCoordinator createCoordinator( - TableSchema tableSchema, Map tableProperties) { - ObjectMapper mapper = new ObjectMapper(); - IcebergSinkConfig config = mapper.convertValue(tableProperties, IcebergSinkConfig.class); - String warehousePath = getWarehousePath(config); - config.setWarehousePath(warehousePath); - - String scheme = UrlParser.parseLocationScheme(warehousePath); - TableIdentifier tableIdentifier = - TableIdentifier.of(config.getDatabaseName(), config.getTableName()); - Configuration hadoopConf = createHadoopConf(scheme, config); - - try (HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); ) { - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - return new IcebergSinkCoordinator(icebergTable); - } catch (Exception e) { - throw Status.FAILED_PRECONDITION - .withDescription( - String.format("failed to load iceberg table: %s", e.getMessage())) - .withCause(e) - .asRuntimeException(); - } - } - - @Override - public SinkWriter createWriter(TableSchema tableSchema, Map tableProperties) { - ObjectMapper mapper = new ObjectMapper(); - IcebergSinkConfig config = mapper.convertValue(tableProperties, IcebergSinkConfig.class); - String warehousePath = getWarehousePath(config); - config.setWarehousePath(warehousePath); - - String scheme = UrlParser.parseLocationScheme(warehousePath); - TableIdentifier tableIdentifier = - TableIdentifier.of(config.getDatabaseName(), config.getTableName()); - Configuration hadoopConf = createHadoopConf(scheme, config); - SinkWriter sink = null; - - try (HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); ) { - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - String sinkType = config.getSinkType(); - if (sinkType.equals("append-only")) { - sink = - new AppendOnlyIcebergSinkWriter( - tableSchema, hadoopCatalog, icebergTable, FILE_FORMAT); - } else if (sinkType.equals("upsert")) { - sink = - new UpsertIcebergSinkWriter( - tableSchema, hadoopCatalog, - icebergTable, FILE_FORMAT); - } - } catch (Exception e) { - throw Status.FAILED_PRECONDITION - .withDescription( - String.format("failed to load iceberg table: %s", e.getMessage())) - .withCause(e) - .asRuntimeException(); - } - - if (sink == null) { - throw UNIMPLEMENTED - .withDescription("unsupported mode: " + config.getSinkType()) - .asRuntimeException(); - } - return sink; - } - - @Override - public void validate( - TableSchema tableSchema, Map tableProperties, SinkType sinkType) { - ObjectMapper mapper = new ObjectMapper(); - mapper.configure(DeserializationFeature.FAIL_ON_MISSING_CREATOR_PROPERTIES, true); - IcebergSinkConfig config = mapper.convertValue(tableProperties, IcebergSinkConfig.class); - - String warehousePath = getWarehousePath(config); - String scheme = UrlParser.parseLocationScheme(warehousePath); - TableIdentifier tableIdentifier = - TableIdentifier.of(config.getDatabaseName(), config.getTableName()); - Configuration hadoopConf = createHadoopConf(scheme, config); - - try (HadoopCatalog hadoopCatalog = new HadoopCatalog(hadoopConf, warehousePath); ) { - Table icebergTable = hadoopCatalog.loadTable(tableIdentifier); - IcebergSinkUtil.checkSchema(tableSchema, icebergTable.schema()); - } catch (Exception e) { - throw Status.INTERNAL - .withDescription( - String.format("failed to load iceberg table: %s", e.getMessage())) - .withCause(e) - .asRuntimeException(); - } - - if (!config.getSinkType().equals("append-only") && !config.getSinkType().equals("upsert")) { - throw UNIMPLEMENTED - .withDescription("unsupported mode: " + config.getSinkType()) - .asRuntimeException(); - } - - switch (sinkType) { - case SINK_TYPE_UPSERT: - // For upsert iceberg sink, the user must specify its primary key explicitly. - if (tableSchema.getPrimaryKeys().isEmpty()) { - throw Status.INVALID_ARGUMENT - .withDescription("please define primary key for upsert iceberg sink") - .asRuntimeException(); - } - break; - case SINK_TYPE_APPEND_ONLY: - case SINK_TYPE_FORCE_APPEND_ONLY: - break; - default: - throw Status.INTERNAL.asRuntimeException(); - } - } - - private static String getWarehousePath(IcebergSinkConfig config) { - String warehousePath = config.getWarehousePath(); - // unify s3 and s3a - if (warehousePath.startsWith("s3://")) { - return warehousePath.replace("s3://", "s3a://"); - } - return warehousePath; - } - - private Configuration createHadoopConf(String scheme, IcebergSinkConfig config) { - switch (scheme) { - case "file": - return new Configuration(); - case "s3a": - Configuration hadoopConf = S3Utils.getHadoopConf(config); - hadoopConf.set(confIoImpl, s3FileIOImpl); - return hadoopConf; - default: - throw UNIMPLEMENTED - .withDescription( - String.format("scheme %s not supported for warehouse path", scheme)) - .asRuntimeException(); - } - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkUtil.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkUtil.java deleted file mode 100644 index 392ea0f4a39f3..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkUtil.java +++ /dev/null @@ -1,112 +0,0 @@ -/* - * Copyright 2024 RisingWave Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.risingwave.connector; - -import static io.grpc.Status.INVALID_ARGUMENT; -import static io.grpc.Status.UNIMPLEMENTED; - -import com.risingwave.connector.api.TableSchema; -import com.risingwave.proto.Data; -import io.grpc.Status; -import java.util.List; -import java.util.Map; -import org.apache.iceberg.Schema; -import org.apache.iceberg.types.Type; -import org.apache.iceberg.types.Types; - -public class IcebergSinkUtil { - private static Type convertType(Data.DataType.TypeName typeName) { - switch (typeName) { - case INT16: - case INT32: - return Types.IntegerType.get(); - case INT64: - return Types.LongType.get(); - case FLOAT: - return Types.FloatType.get(); - case DOUBLE: - return Types.DoubleType.get(); - case BOOLEAN: - return Types.BooleanType.get(); - case VARCHAR: - return Types.StringType.get(); - case DECIMAL: - return Types.DecimalType.of(10, 0); - case TIMESTAMP: - return Types.TimestampType.withoutZone(); - case TIMESTAMPTZ: - return Types.TimestampType.withZone(); - case DATE: - return Types.DateType.get(); - case TIME: - return Types.TimeType.get(); - case STRUCT: - case LIST: - throw UNIMPLEMENTED - .withDescription(String.format("not support %s now", typeName)) - .asRuntimeException(); - case INTERVAL: - throw INVALID_ARGUMENT - .withDescription(String.format("Illegal type %s in Iceberg", typeName)) - .asRuntimeException(); - default: - throw INVALID_ARGUMENT - .withDescription("unspecified type" + typeName) - .asRuntimeException(); - } - } - - public static void checkSchema(TableSchema tableSchema, Schema icebergSchema) { - if (icebergSchema == null) { - throw Status.FAILED_PRECONDITION - .withDescription("Schema of iceberg table is null") - .asRuntimeException(); - } - Map tableColumnTypes = tableSchema.getColumnTypes(); - List icebergNestedFields = icebergSchema.columns(); - // Check that all columns in tableSchema exist in the iceberg table and that existing column - // types match. - for (Map.Entry tableColumnEntry : - tableColumnTypes.entrySet()) { - Types.NestedField field = icebergSchema.findField(tableColumnEntry.getKey()); - if (field == null) { - throw Status.FAILED_PRECONDITION - .withDescription( - String.format( - "The name of table schema does not match. Column name: %s.", - tableColumnEntry.getKey())) - .asRuntimeException(); - } - if (!convertType(tableColumnEntry.getValue()).equals(field.type())) { - throw Status.FAILED_PRECONDITION - .withDescription( - String.format( - "The type of table schema does not match. Column name: %s.", - tableColumnEntry.getKey())) - .asRuntimeException(); - } - } - // Check that all required columns in the iceberg table exist in tableSchema. - for (Types.NestedField field : icebergNestedFields) { - if (tableColumnTypes.get(field.name()) == null) { - throw Status.FAILED_PRECONDITION - .withDescription(String.format("missing a required field %s", field.name())) - .asRuntimeException(); - } - } - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkWriterBase.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkWriterBase.java deleted file mode 100644 index e42030c217bfc..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/IcebergSinkWriterBase.java +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright 2024 RisingWave Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package com.risingwave.connector; - -import com.google.protobuf.ByteString; -import com.risingwave.connector.api.TableSchema; -import com.risingwave.connector.api.sink.SinkWriter; -import com.risingwave.java.utils.ObjectSerde; -import com.risingwave.proto.ConnectorServiceProto; -import java.util.Optional; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.hadoop.HadoopCatalog; - -public abstract class IcebergSinkWriterBase implements SinkWriter { - protected final TableSchema tableSchema; - protected final HadoopCatalog hadoopCatalog; - protected final Table icebergTable; - protected final Schema rowSchema; - protected final FileFormat fileFormat; - - public IcebergSinkWriterBase( - TableSchema tableSchema, - Table icebergTable, - HadoopCatalog hadoopCatalog, - Schema rowSchema, - FileFormat fileFormat) { - this.tableSchema = tableSchema; - this.hadoopCatalog = hadoopCatalog; - this.rowSchema = rowSchema; - this.icebergTable = icebergTable; - this.fileFormat = fileFormat; - } - - @Override - public void beginEpoch(long epoch) {} - - protected abstract IcebergMetadata collectSinkMetadata(); - - @Override - public Optional barrier(boolean isCheckpoint) { - if (isCheckpoint) { - IcebergMetadata metadata = collectSinkMetadata(); - return Optional.of( - ConnectorServiceProto.SinkMetadata.newBuilder() - .setSerialized( - ConnectorServiceProto.SinkMetadata.SerializedMetadata - .newBuilder() - .setMetadata( - ByteString.copyFrom( - ObjectSerde.serializeObject(metadata))) - .build()) - .build()); - } else { - return Optional.empty(); - } - } - - public HadoopCatalog getHadoopCatalog() { - return this.hadoopCatalog; - } - - public Table getIcebergTable() { - return this.icebergTable; - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/SinkRowMap.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/SinkRowMap.java deleted file mode 100644 index aba4d96f495f8..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/SinkRowMap.java +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector; - -import com.risingwave.connector.api.PkComparator; -import com.risingwave.connector.api.sink.SinkRow; -import io.grpc.Status; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; -import org.apache.iceberg.data.Record; - -public class SinkRowMap { - TreeMap>, SinkRowOp> map = new TreeMap<>(new PkComparator()); - - public void clear() { - map.clear(); - } - - public void insert(List> key, Record row) { - if (!map.containsKey(key)) { - map.put(key, SinkRowOp.insertOp(row)); - } else { - SinkRowOp sinkRowOp = map.get(key); - if (sinkRowOp.isDelete()) { - map.put(key, SinkRowOp.updateOp(sinkRowOp.getDelete(), row)); - } else { - throw Status.FAILED_PRECONDITION - .withDescription("try to insert a duplicated primary key") - .asRuntimeException(); - } - } - } - - public void delete(List> key, Record row) { - if (!map.containsKey(key)) { - map.put(key, SinkRowOp.deleteOp(row)); - } else { - SinkRowOp sinkRowOp = map.get(key); - Record insert = sinkRowOp.getInsert(); - if (insert == null) { - throw Status.FAILED_PRECONDITION - .withDescription("try to double delete a primary key") - .asRuntimeException(); - } - // TODO: may enable it again - // assertRowValuesEqual(insert, row); - Record delete = sinkRowOp.getDelete(); - if (delete != null) { - map.put(key, SinkRowOp.deleteOp(delete)); - } else { - map.remove(key); - } - } - } - - public Map>, SinkRowOp> getMap() { - return map; - } - - private void assertRowValuesEqual(SinkRow insert, SinkRow delete) { - for (int i = 0; i < delete.size(); i++) { - if (!insert.get(i).equals(delete.get(i))) { - throw Status.FAILED_PRECONDITION - .withDescription( - "delete row value " - + delete.get(i) - + " does not match inserted value " - + insert.get(i)) - .asRuntimeException(); - } - } - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/SinkRowOp.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/SinkRowOp.java deleted file mode 100644 index 2addfe01a486b..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/SinkRowOp.java +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector; - -import io.grpc.Status; -import org.apache.iceberg.data.Record; - -public class SinkRowOp { - private final Record delete; - private final Record insert; - - public static SinkRowOp insertOp(Record row) { - if (row == null) { - throw Status.FAILED_PRECONDITION - .withDescription("row op must not be null to initialize insertOp") - .asRuntimeException(); - } - return new SinkRowOp(null, row); - } - - public static SinkRowOp deleteOp(Record row) { - if (row == null) { - throw Status.FAILED_PRECONDITION - .withDescription("row op must not be null to initialize deleteOp") - .asRuntimeException(); - } - return new SinkRowOp(row, null); - } - - public static SinkRowOp updateOp(Record delete, Record insert) { - if (delete == null || insert == null) { - throw Status.FAILED_PRECONDITION - .withDescription("row ops must not be null initialize updateOp") - .asRuntimeException(); - } - return new SinkRowOp(delete, insert); - } - - private SinkRowOp(Record delete, Record insert) { - this.delete = delete; - this.insert = insert; - } - - public boolean isDelete() { - return insert == null && delete != null; - } - - public Record getDelete() { - return delete; - } - - public Record getInsert() { - return insert; - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/UpsertIcebergSinkWriter.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/UpsertIcebergSinkWriter.java deleted file mode 100644 index aae9bdf46c4c2..0000000000000 --- a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/UpsertIcebergSinkWriter.java +++ /dev/null @@ -1,248 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package com.risingwave.connector; - -import static io.grpc.Status.INTERNAL; -import static io.grpc.Status.UNIMPLEMENTED; - -import com.risingwave.connector.api.TableSchema; -import com.risingwave.connector.api.sink.SinkRow; -import io.grpc.Status; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.stream.Collectors; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; -import org.apache.iceberg.FileFormat; -import org.apache.iceberg.PartitionKey; -import org.apache.iceberg.Schema; -import org.apache.iceberg.Table; -import org.apache.iceberg.data.GenericRecord; -import org.apache.iceberg.data.Record; -import org.apache.iceberg.data.parquet.GenericParquetWriter; -import org.apache.iceberg.deletes.EqualityDeleteWriter; -import org.apache.iceberg.hadoop.HadoopCatalog; -import org.apache.iceberg.io.DataWriter; -import org.apache.iceberg.io.OutputFile; -import org.apache.iceberg.parquet.Parquet; -import org.apache.iceberg.types.Types; - -public class UpsertIcebergSinkWriter extends IcebergSinkWriterBase { - private final Schema deleteRowSchema; - private final List pkIndices; - private boolean closed = false; - private boolean updateBufferExists = false; - private Map sinkRowMapByPartition = new HashMap<>(); - - public UpsertIcebergSinkWriter( - TableSchema tableSchema, - HadoopCatalog hadoopCatalog, - Table icebergTable, - FileFormat fileFormat) { - super( - tableSchema, - icebergTable, - hadoopCatalog, - icebergTable.schema().select(Arrays.asList(tableSchema.getColumnNames())), - fileFormat); - this.deleteRowSchema = icebergTable.schema().select(tableSchema.getPrimaryKeys()); - this.pkIndices = - tableSchema.getPrimaryKeys().stream() - .map(columnName -> tableSchema.getColumnIndex(columnName)) - .collect(Collectors.toList()); - } - - private static Record newRecord(Schema schema, SinkRow row) { - Record record = GenericRecord.create(schema); - for (int i = 0; i < schema.columns().size(); i++) { - record.set(i, row.get(i)); - } - return record; - } - - private EqualityDeleteWriter newEqualityDeleteWriter(PartitionKey partitionKey) { - try { - String filename = fileFormat.addExtension(UUID.randomUUID().toString()); - OutputFile outputFile = - icebergTable - .io() - .newOutputFile( - icebergTable.location() - + "/data/" - + icebergTable.spec().partitionToPath(partitionKey) - + "/" - + filename); - return Parquet.writeDeletes(outputFile) - .forTable(icebergTable) - .rowSchema(deleteRowSchema) - .withSpec(icebergTable.spec()) - .withPartition(partitionKey) - .createWriterFunc(GenericParquetWriter::buildWriter) - .overwrite() - .equalityFieldIds( - deleteRowSchema.columns().stream() - .mapToInt(Types.NestedField::fieldId) - .toArray()) - .buildEqualityWriter(); - } catch (Exception e) { - throw INTERNAL.withDescription("failed to create outputFile and equalityDeleteWriter") - .asRuntimeException(); - } - } - - private DataWriter newDataWriter(PartitionKey partitionKey) { - try { - String filename = fileFormat.addExtension(UUID.randomUUID().toString()); - OutputFile outputFile = - icebergTable - .io() - .newOutputFile( - icebergTable.location() - + "/data/" - + icebergTable.spec().partitionToPath(partitionKey) - + "/" - + filename); - return Parquet.writeData(outputFile) - .schema(rowSchema) - .withSpec(icebergTable.spec()) - .withPartition(partitionKey) - .createWriterFunc(GenericParquetWriter::buildWriter) - .overwrite() - .build(); - } catch (Exception e) { - throw INTERNAL.withDescription("failed to create outputFile and dataWriter") - .asRuntimeException(); - } - } - - private List> getKeyFromRow(SinkRow row) { - return this.pkIndices.stream() - .map(idx -> (Comparable) row.get(idx)) - .collect(Collectors.toList()); - } - - @Override - public boolean write(Iterable rows) { - for (SinkRow row : rows) { - if (row.size() != tableSchema.getColumnNames().length) { - throw Status.FAILED_PRECONDITION - .withDescription("row values do not match table schema") - .asRuntimeException(); - } - Record record = newRecord(rowSchema, row); - PartitionKey partitionKey = - new PartitionKey(icebergTable.spec(), icebergTable.schema()); - partitionKey.partition(record); - SinkRowMap sinkRowMap; - if (sinkRowMapByPartition.containsKey(partitionKey)) { - sinkRowMap = sinkRowMapByPartition.get(partitionKey); - } else { - sinkRowMap = new SinkRowMap(); - sinkRowMapByPartition.put(partitionKey, sinkRowMap); - } - switch (row.getOp()) { - case INSERT: - sinkRowMap.insert(getKeyFromRow(row), newRecord(rowSchema, row)); - break; - case DELETE: - sinkRowMap.delete(getKeyFromRow(row), newRecord(deleteRowSchema, row)); - break; - case UPDATE_DELETE: - if (updateBufferExists) { - throw Status.FAILED_PRECONDITION - .withDescription("an UPDATE_INSERT should precede an UPDATE_DELETE") - .asRuntimeException(); - } - sinkRowMap.delete(getKeyFromRow(row), newRecord(deleteRowSchema, row)); - updateBufferExists = true; - break; - case UPDATE_INSERT: - if (!updateBufferExists) { - throw Status.FAILED_PRECONDITION - .withDescription("an UPDATE_INSERT should precede an UPDATE_DELETE") - .asRuntimeException(); - } - sinkRowMap.insert(getKeyFromRow(row), newRecord(rowSchema, row)); - updateBufferExists = false; - break; - default: - throw UNIMPLEMENTED - .withDescription("unsupported operation: " + row.getOp()) - .asRuntimeException(); - } - } - return false; - } - - @Override - protected IcebergMetadata collectSinkMetadata() { - List dataFileList = new ArrayList<>(); - List deleteFileList = new ArrayList<>(); - for (Map.Entry entry : sinkRowMapByPartition.entrySet()) { - EqualityDeleteWriter equalityDeleteWriter = - newEqualityDeleteWriter(entry.getKey()); - DataWriter dataWriter = newDataWriter(entry.getKey()); - for (SinkRowOp sinkRowOp : entry.getValue().map.values()) { - Record insert = sinkRowOp.getInsert(); - Record delete = sinkRowOp.getDelete(); - if (insert != null) { - dataWriter.write(insert); - } - if (delete != null) { - equalityDeleteWriter.write(delete); - } - } - try { - equalityDeleteWriter.close(); - dataWriter.close(); - } catch (IOException e) { - throw INTERNAL.withDescription( - "failed to close dataWriter and equalityDeleteWriter") - .asRuntimeException(); - } - - if (equalityDeleteWriter.length() > 0) { - DeleteFile eqDeletes = equalityDeleteWriter.toDeleteFile(); - deleteFileList.add(eqDeletes); - } - if (dataWriter.length() > 0) { - DataFile dataFile = dataWriter.toDataFile(); - dataFileList.add(dataFile); - } - } - sinkRowMapByPartition.clear(); - return new IcebergMetadata( - dataFileList.toArray(new DataFile[0]), deleteFileList.toArray(new DeleteFile[0])); - } - - @Override - public void drop() { - try { - hadoopCatalog.close(); - closed = true; - } catch (Exception e) { - throw INTERNAL.withCause(e).asRuntimeException(); - } - } - - public boolean isClosed() { - return closed; - } -} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/JniCatalogWrapper.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/JniCatalogWrapper.java new file mode 100644 index 0000000000000..e8c900b37e88d --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/JniCatalogWrapper.java @@ -0,0 +1,97 @@ +/* + * Copyright 2024 RisingWave Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.risingwave.connector.catalog; + +import static com.google.common.base.Preconditions.checkArgument; + +import java.util.HashMap; +import java.util.Objects; +import org.apache.iceberg.CatalogUtil; +import org.apache.iceberg.catalog.Catalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.rest.CatalogHandlers; +import org.apache.iceberg.rest.requests.UpdateTableRequest; +import org.apache.iceberg.rest.responses.LoadTableResponse; + +/** This class provide jni interface to iceberg catalog. */ +public class JniCatalogWrapper { + private final Catalog catalog; + + JniCatalogWrapper(Catalog catalog) { + this.catalog = Objects.requireNonNull(catalog, "Catalog can't be null!"); + } + + /** + * Load table through this prox. + * + * @param tableIdentifier Table identifier. + * @return Response serialized using json. + * @throws Exception + */ + public String loadTable(String tableIdentifier) throws Exception { + TableIdentifier id = TableIdentifier.parse(tableIdentifier); + LoadTableResponse resp = CatalogHandlers.loadTable(catalog, id); + return RESTObjectMapper.mapper().writer().writeValueAsString(resp); + } + + /** + * Update table through this prox. + * + * @param updateTableRequest Request serialized using json. + * @return Response serialized using json. + * @throws Exception + */ + public String updateTable(String updateTableRequest) throws Exception { + UpdateTableRequest req = + RESTObjectMapper.mapper().readValue(updateTableRequest, UpdateTableRequest.class); + LoadTableResponse resp = CatalogHandlers.updateTable(catalog, req.identifier(), req); + return RESTObjectMapper.mapper().writer().writeValueAsString(resp); + } + + /** + * Create JniCatalogWrapper instance. + * + * @param name Catalog name. + * @param klassName Delegated catalog class name. + * @param props Catalog properties. + * @return JniCatalogWrapper instance. + */ + public static JniCatalogWrapper create(String name, String klassName, String[] props) { + checkArgument( + props.length % 2 == 0, + "props should be key-value pairs, but length is: " + props.length); + + // Thread.currentThread().setContextClassLoader(ClassLoader.getSystemClassLoader()); + System.out.println("Current thread name is: " + Thread.currentThread().getName()); + + // try { + // Thread.currentThread().getContextClassLoader().loadClass(klassName); + // } catch (ClassNotFoundException e) { + // throw new RuntimeException(e); + // } + try { + HashMap config = new HashMap<>(props.length / 2); + for (int i = 0; i < props.length; i += 2) { + config.put(props[i], props[i + 1]); + } + Catalog catalog = CatalogUtil.loadCatalog(klassName, name, config, null); + return new JniCatalogWrapper(catalog); + } catch (Exception e) { + throw new RuntimeException(e); + } + } +} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/RESTObjectMapper.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/RESTObjectMapper.java new file mode 100644 index 0000000000000..a4341ad942396 --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/com/risingwave/connector/catalog/RESTObjectMapper.java @@ -0,0 +1,50 @@ +/* + * Copyright 2024 RisingWave Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.risingwave.connector.catalog; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.PropertyAccessor; +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.PropertyNamingStrategy; +import org.apache.iceberg.rest.RESTSerializers; + +class RESTObjectMapper { + private static final JsonFactory FACTORY = new JsonFactory(); + private static final ObjectMapper MAPPER = new ObjectMapper(FACTORY); + private static volatile boolean isInitialized = false; + + private RESTObjectMapper() {} + + static ObjectMapper mapper() { + if (!isInitialized) { + synchronized (RESTObjectMapper.class) { + if (!isInitialized) { + MAPPER.setVisibility(PropertyAccessor.FIELD, JsonAutoDetect.Visibility.ANY); + MAPPER.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + MAPPER.setPropertyNamingStrategy( + new PropertyNamingStrategy.KebabCaseStrategy()); + RESTSerializers.registerAll(MAPPER); + isInitialized = true; + } + } + } + + return MAPPER; + } +} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynClasses.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynClasses.java new file mode 100644 index 0000000000000..65a75e774d2ff --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynClasses.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.common; + +import org.apache.iceberg.relocated.com.google.common.base.Joiner; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; + +import java.util.Set; + +public class DynClasses { + + private DynClasses() {} + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private Class foundClass = null; + private boolean nullOk = false; + private Set classNames = Sets.newLinkedHashSet(); + + private Builder() {} + + /** + * Set the {@link ClassLoader} used to lookup classes by name. + * + *

If not set, the current thread's ClassLoader is used. + * + * @param newLoader a ClassLoader + * @return this Builder for method chaining + */ + public Builder loader(ClassLoader newLoader) { + return this; + } + + /** + * Checks for an implementation of the class by name. + * + * @param className name of a class + * @return this Builder for method chaining + */ + public Builder impl(String className) { + classNames.add(className); + + if (foundClass != null) { + return this; + } + + try { + this.foundClass = Class.forName(className); + } catch (ClassNotFoundException e) { + // not the right implementation + } + + return this; + } + + /** + * Instructs this builder to return null if no class is found, rather than throwing an + * Exception. + * + * @return this Builder for method chaining + */ + public Builder orNull() { + this.nullOk = true; + return this; + } + + /** + * Returns the first implementation or throws ClassNotFoundException if one was not found. + * + * @param Java superclass + * @return a {@link Class} for the first implementation found + * @throws ClassNotFoundException if no implementation was found + */ + @SuppressWarnings("unchecked") + public Class buildChecked() throws ClassNotFoundException { + if (!nullOk && foundClass == null) { + throw new ClassNotFoundException( + "Cannot find class; alternatives: " + Joiner.on(", ").join(classNames)); + } + return (Class) foundClass; + } + + /** + * Returns the first implementation or throws RuntimeException if one was not found. + * + * @param Java superclass + * @return a {@link Class} for the first implementation found + * @throws RuntimeException if no implementation was found + */ + @SuppressWarnings("unchecked") + public Class build() { + if (!nullOk && foundClass == null) { + throw new RuntimeException( + "Cannot find class; alternatives: " + Joiner.on(", ").join(classNames)); + } + return (Class) foundClass; + } + } +} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynConstructors.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynConstructors.java new file mode 100644 index 0000000000000..61566f4e191ff --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynConstructors.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.common; + +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Throwables; +import org.apache.iceberg.relocated.com.google.common.collect.Maps; + +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Arrays; +import java.util.Map; + +/** Copied from parquet-common */ +public class DynConstructors { + + private DynConstructors() {} + + public static class Ctor extends DynMethods.UnboundMethod { + private final Constructor ctor; + private final Class constructed; + + private Ctor(Constructor constructor, Class constructed) { + super(null, "newInstance"); + this.ctor = constructor; + this.constructed = constructed; + } + + public Class getConstructedClass() { + return constructed; + } + + public C newInstanceChecked(Object... args) throws Exception { + try { + if (args.length > ctor.getParameterCount()) { + return ctor.newInstance(Arrays.copyOfRange(args, 0, ctor.getParameterCount())); + } else { + return ctor.newInstance(args); + } + } catch (InstantiationException | IllegalAccessException e) { + throw e; + } catch (InvocationTargetException e) { + Throwables.propagateIfInstanceOf(e.getCause(), Exception.class); + Throwables.propagateIfInstanceOf(e.getCause(), RuntimeException.class); + throw Throwables.propagate(e.getCause()); + } + } + + public C newInstance(Object... args) { + try { + return newInstanceChecked(args); + } catch (Exception e) { + Throwables.propagateIfInstanceOf(e, RuntimeException.class); + throw Throwables.propagate(e); + } + } + + @Override + @SuppressWarnings("unchecked") + public R invoke(Object target, Object... args) { + Preconditions.checkArgument( + target == null, "Invalid call to constructor: target must be null"); + return (R) newInstance(args); + } + + @Override + @SuppressWarnings("unchecked") + public R invokeChecked(Object target, Object... args) throws Exception { + Preconditions.checkArgument( + target == null, "Invalid call to constructor: target must be null"); + return (R) newInstanceChecked(args); + } + + @Override + public DynMethods.BoundMethod bind(Object receiver) { + throw new IllegalStateException("Cannot bind constructors"); + } + + @Override + public boolean isStatic() { + return true; + } + + @Override + public String toString() { + return getClass().getSimpleName() + "(constructor=" + ctor + ", class=" + constructed + ")"; + } + } + + public static Builder builder() { + return new Builder(); + } + + public static Builder builder(Class baseClass) { + return new Builder(baseClass); + } + + public static class Builder { + private final Class baseClass; + private Ctor ctor = null; + private Map problems = Maps.newHashMap(); + + public Builder(Class baseClass) { + this.baseClass = baseClass; + } + + public Builder() { + this.baseClass = null; + } + + /** + * Set the {@link ClassLoader} used to lookup classes by name. + * + *

If not set, the current thread's ClassLoader is used. + * + * @param newLoader a ClassLoader + * @return this Builder for method chaining + */ + public Builder loader(ClassLoader newLoader) { + return this; + } + + public Builder impl(String className, Class... types) { + // don't do any work if an implementation has been found + if (ctor != null) { + return this; + } + + try { + Class targetClass = Class.forName(className); + impl(targetClass, types); + } catch (NoClassDefFoundError | ClassNotFoundException e) { + // cannot load this implementation + problems.put(className, e); + } + return this; + } + + public Builder impl(Class targetClass, Class... types) { + // don't do any work if an implementation has been found + if (ctor != null) { + return this; + } + + try { + ctor = new Ctor(targetClass.getConstructor(types), targetClass); + } catch (NoSuchMethodException e) { + // not the right implementation + problems.put(methodName(targetClass, types), e); + } + return this; + } + + public Builder hiddenImpl(Class... types) { + hiddenImpl(baseClass, types); + return this; + } + + @SuppressWarnings("unchecked") + public Builder hiddenImpl(String className, Class... types) { + // don't do any work if an implementation has been found + if (ctor != null) { + return this; + } + + try { + Class targetClass = Class.forName(className); + hiddenImpl(targetClass, types); + } catch (NoClassDefFoundError | ClassNotFoundException e) { + // cannot load this implementation + problems.put(className, e); + } + return this; + } + + public Builder hiddenImpl(Class targetClass, Class... types) { + // don't do any work if an implementation has been found + if (ctor != null) { + return this; + } + + try { + Constructor hidden = targetClass.getDeclaredConstructor(types); + AccessController.doPrivileged(new MakeAccessible(hidden)); + ctor = new Ctor(hidden, targetClass); + } catch (SecurityException e) { + // unusable + problems.put(methodName(targetClass, types), e); + } catch (NoSuchMethodException e) { + // not the right implementation + problems.put(methodName(targetClass, types), e); + } + return this; + } + + @SuppressWarnings("unchecked") + public Ctor buildChecked() throws NoSuchMethodException { + if (ctor != null) { + return ctor; + } + throw buildCheckedException(baseClass, problems); + } + + @SuppressWarnings("unchecked") + public Ctor build() { + if (ctor != null) { + return ctor; + } + throw buildRuntimeException(baseClass, problems); + } + } + + private static class MakeAccessible implements PrivilegedAction { + private Constructor hidden; + + MakeAccessible(Constructor hidden) { + this.hidden = hidden; + } + + @Override + public Void run() { + hidden.setAccessible(true); + return null; + } + } + + private static NoSuchMethodException buildCheckedException( + Class baseClass, Map problems) { + NoSuchMethodException exc = + new NoSuchMethodException( + "Cannot find constructor for " + baseClass + "\n" + formatProblems(problems)); + problems.values().forEach(exc::addSuppressed); + return exc; + } + + private static RuntimeException buildRuntimeException( + Class baseClass, Map problems) { + RuntimeException exc = + new RuntimeException( + "Cannot find constructor for " + baseClass + "\n" + formatProblems(problems)); + problems.values().forEach(exc::addSuppressed); + return exc; + } + + private static String formatProblems(Map problems) { + StringBuilder sb = new StringBuilder(); + boolean first = true; + for (Map.Entry problem : problems.entrySet()) { + if (first) { + first = false; + } else { + sb.append("\n"); + } + sb.append("\tMissing ") + .append(problem.getKey()) + .append(" [") + .append(problem.getValue().getClass().getName()) + .append(": ") + .append(problem.getValue().getMessage()) + .append("]"); + } + return sb.toString(); + } + + private static String methodName(Class targetClass, Class... types) { + StringBuilder sb = new StringBuilder(); + sb.append(targetClass.getName()).append("("); + boolean first = true; + for (Class type : types) { + if (first) { + first = false; + } else { + sb.append(","); + } + sb.append(type.getName()); + } + sb.append(")"); + return sb.toString(); + } +} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynFields.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynFields.java new file mode 100644 index 0000000000000..80b6af8cb7f93 --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynFields.java @@ -0,0 +1,428 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.common; + +import org.apache.iceberg.relocated.com.google.common.base.Joiner; +import org.apache.iceberg.relocated.com.google.common.base.MoreObjects; +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Throwables; +import org.apache.iceberg.relocated.com.google.common.collect.Sets; + +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Set; + +public class DynFields { + + private DynFields() {} + + /** + * Convenience wrapper class around {@link Field}. + * + *

Allows callers to invoke the wrapped method with all Exceptions wrapped by RuntimeException, + * or with a single Exception catch block. + */ + public static class UnboundField { + private final Field field; + private final String name; + + private UnboundField(Field field, String name) { + this.field = field; + this.name = name; + } + + @SuppressWarnings("unchecked") + public T get(Object target) { + try { + return (T) field.get(target); + } catch (IllegalAccessException e) { + throw Throwables.propagate(e); + } + } + + public void set(Object target, T value) { + try { + field.set(target, value); + } catch (IllegalAccessException e) { + throw Throwables.propagate(e); + } + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("class", field.getDeclaringClass().toString()) + .add("name", name) + .add("type", field.getType()) + .toString(); + } + + /** + * Returns this method as a BoundMethod for the given receiver. + * + * @param target an Object on which to get or set this field + * @return a {@link BoundField} for this field and the target + * @throws IllegalStateException if the method is static + * @throws IllegalArgumentException if the receiver's class is incompatible + */ + public BoundField bind(Object target) { + Preconditions.checkState( + !isStatic() || this == AlwaysNull.INSTANCE, "Cannot bind static field %s", name); + Preconditions.checkArgument( + field.getDeclaringClass().isAssignableFrom(target.getClass()), + "Cannot bind field %s to instance of %s", + name, + target.getClass()); + + return new BoundField<>(this, target); + } + + /** + * Returns this field as a StaticField. + * + * @return a {@link StaticField} for this field + * @throws IllegalStateException if the method is not static + */ + public StaticField asStatic() { + Preconditions.checkState(isStatic(), "Field %s is not static", name); + return new StaticField<>(this); + } + + /** Returns whether the field is a static field. */ + public boolean isStatic() { + return Modifier.isStatic(field.getModifiers()); + } + + /** Returns whether the field is always null. */ + public boolean isAlwaysNull() { + return this == AlwaysNull.INSTANCE; + } + } + + private static class AlwaysNull extends UnboundField { + private static final AlwaysNull INSTANCE = new AlwaysNull(); + + private AlwaysNull() { + super(null, "AlwaysNull"); + } + + @Override + public Void get(Object target) { + return null; + } + + @Override + public void set(Object target, Void value) {} + + @Override + public String toString() { + return "Field(AlwaysNull)"; + } + + @Override + public boolean isStatic() { + return true; + } + + @Override + public boolean isAlwaysNull() { + return true; + } + } + + public static class StaticField { + private final UnboundField field; + + private StaticField(UnboundField field) { + this.field = field; + } + + public T get() { + return field.get(null); + } + + public void set(T value) { + field.set(null, value); + } + } + + public static class BoundField { + private final UnboundField field; + private final Object target; + + private BoundField(UnboundField field, Object target) { + this.field = field; + this.target = target; + } + + public T get() { + return field.get(target); + } + + public void set(T value) { + field.set(target, value); + } + } + + public static Builder builder() { + return new Builder(); + } + + public static class Builder { + private UnboundField field = null; + private final Set candidates = Sets.newHashSet(); + private boolean defaultAlwaysNull = false; + + private Builder() {} + + /** + * Set the {@link ClassLoader} used to lookup classes by name. + * + *

If not set, the current thread's ClassLoader is used. + * + * @param newLoader a ClassLoader + * @return this Builder for method chaining + */ + public Builder loader(ClassLoader newLoader) { + return this; + } + + /** + * Instructs this builder to return AlwaysNull if no implementation is found. + * + * @return this Builder for method chaining + */ + public Builder defaultAlwaysNull() { + this.defaultAlwaysNull = true; + return this; + } + + /** + * Checks for an implementation, first finding the class by name. + * + * @param className name of a class + * @param fieldName name of the field + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getField(String) + */ + public Builder impl(String className, String fieldName) { + // don't do any work if an implementation has been found + if (field != null) { + return this; + } + + try { + Class targetClass = Class.forName(className); + impl(targetClass, fieldName); + } catch (ClassNotFoundException e) { + // not the right implementation + candidates.add(className + "." + fieldName); + } + return this; + } + + /** + * Checks for an implementation. + * + * @param targetClass a class instance + * @param fieldName name of a field (different from constructor) + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getField(String) + */ + public Builder impl(Class targetClass, String fieldName) { + // don't do any work if an implementation has been found + if (field != null || targetClass == null) { + return this; + } + + try { + this.field = new UnboundField<>(targetClass.getField(fieldName), fieldName); + } catch (NoSuchFieldException e) { + // not the right implementation + candidates.add(targetClass.getName() + "." + fieldName); + } + return this; + } + + /** + * Checks for a hidden implementation, first finding the class by name. + * + * @param className name of a class + * @param fieldName name of a field (different from constructor) + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getField(String) + */ + public Builder hiddenImpl(String className, String fieldName) { + // don't do any work if an implementation has been found + if (field != null) { + return this; + } + + try { + Class targetClass = Class.forName(className); + hiddenImpl(targetClass, fieldName); + } catch (ClassNotFoundException e) { + // not the right implementation + candidates.add(className + "." + fieldName); + } + return this; + } + + /** + * Checks for a hidden implementation. + * + * @param targetClass a class instance + * @param fieldName name of a field (different from constructor) + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getField(String) + */ + public Builder hiddenImpl(Class targetClass, String fieldName) { + // don't do any work if an implementation has been found + if (field != null || targetClass == null) { + return this; + } + + try { + Field hidden = targetClass.getDeclaredField(fieldName); + AccessController.doPrivileged(new MakeFieldAccessible(hidden)); + this.field = new UnboundField(hidden, fieldName); + } catch (SecurityException | NoSuchFieldException e) { + // unusable + candidates.add(targetClass.getName() + "." + fieldName); + } + return this; + } + + /** + * Returns the first valid implementation as a UnboundField or throws a NoSuchFieldException if + * there is none. + * + * @param Java class stored in the field + * @return a {@link UnboundField} with a valid implementation + * @throws NoSuchFieldException if no implementation was found + */ + @SuppressWarnings("unchecked") + public UnboundField buildChecked() throws NoSuchFieldException { + if (field != null) { + return (UnboundField) field; + } else if (defaultAlwaysNull) { + return (UnboundField) AlwaysNull.INSTANCE; + } else { + throw new NoSuchFieldException( + "Cannot find field from candidates: " + Joiner.on(", ").join(candidates)); + } + } + + /** + * Returns the first valid implementation as a BoundMethod or throws a NoSuchMethodException if + * there is none. + * + * @param target an Object on which to get and set the field + * @param Java class stored in the field + * @return a {@link BoundField} with a valid implementation and target + * @throws IllegalStateException if the method is static + * @throws IllegalArgumentException if the receiver's class is incompatible + * @throws NoSuchFieldException if no implementation was found + */ + public BoundField buildChecked(Object target) throws NoSuchFieldException { + return this.buildChecked().bind(target); + } + + /** + * Returns the first valid implementation as a UnboundField or throws a NoSuchFieldException if + * there is none. + * + * @param Java class stored in the field + * @return a {@link UnboundField} with a valid implementation + * @throws RuntimeException if no implementation was found + */ + @SuppressWarnings("unchecked") + public UnboundField build() { + if (field != null) { + return (UnboundField) field; + } else if (defaultAlwaysNull) { + return (UnboundField) AlwaysNull.INSTANCE; + } else { + throw new RuntimeException( + "Cannot find field from candidates: " + Joiner.on(", ").join(candidates)); + } + } + + /** + * Returns the first valid implementation as a BoundMethod or throws a RuntimeException if there + * is none. + * + * @param target an Object on which to get and set the field + * @param Java class stored in the field + * @return a {@link BoundField} with a valid implementation and target + * @throws IllegalStateException if the method is static + * @throws IllegalArgumentException if the receiver's class is incompatible + * @throws RuntimeException if no implementation was found + */ + public BoundField build(Object target) { + return this.build().bind(target); + } + + /** + * Returns the first valid implementation as a StaticField or throws a NoSuchFieldException if + * there is none. + * + * @param Java class stored in the field + * @return a {@link StaticField} with a valid implementation + * @throws IllegalStateException if the method is not static + * @throws NoSuchFieldException if no implementation was found + */ + public StaticField buildStaticChecked() throws NoSuchFieldException { + return this.buildChecked().asStatic(); + } + + /** + * Returns the first valid implementation as a StaticField or throws a RuntimeException if there + * is none. + * + * @param Java class stored in the field + * @return a {@link StaticField} with a valid implementation + * @throws IllegalStateException if the method is not static + * @throws RuntimeException if no implementation was found + */ + public StaticField buildStatic() { + return this.build().asStatic(); + } + } + + private static class MakeFieldAccessible implements PrivilegedAction { + private Field hidden; + + MakeFieldAccessible(Field hidden) { + this.hidden = hidden; + } + + @Override + public Void run() { + hidden.setAccessible(true); + return null; + } + } +} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynMethods.java b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynMethods.java new file mode 100644 index 0000000000000..281c3d34ed304 --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/DynMethods.java @@ -0,0 +1,522 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iceberg.common; + +import org.apache.iceberg.relocated.com.google.common.base.Preconditions; +import org.apache.iceberg.relocated.com.google.common.base.Throwables; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.security.AccessController; +import java.security.PrivilegedAction; +import java.util.Arrays; + +/** Copied from parquet-common */ +public class DynMethods { + + private DynMethods() {} + + /** + * Convenience wrapper class around {@link Method}. + * + *

Allows callers to invoke the wrapped method with all Exceptions wrapped by RuntimeException, + * or with a single Exception catch block. + */ + public static class UnboundMethod { + + private final Method method; + private final String name; + private final int argLength; + + UnboundMethod(Method method, String name) { + this.method = method; + this.name = name; + this.argLength = + (method == null || method.isVarArgs()) ? -1 : method.getParameterTypes().length; + } + + @SuppressWarnings("unchecked") + public R invokeChecked(Object target, Object... args) throws Exception { + try { + if (argLength < 0) { + return (R) method.invoke(target, args); + } else { + return (R) method.invoke(target, Arrays.copyOfRange(args, 0, argLength)); + } + + } catch (InvocationTargetException e) { + Throwables.propagateIfInstanceOf(e.getCause(), Exception.class); + Throwables.propagateIfInstanceOf(e.getCause(), RuntimeException.class); + throw Throwables.propagate(e.getCause()); + } + } + + public R invoke(Object target, Object... args) { + try { + return this.invokeChecked(target, args); + } catch (Exception e) { + Throwables.propagateIfInstanceOf(e, RuntimeException.class); + throw Throwables.propagate(e); + } + } + + /** + * Returns this method as a BoundMethod for the given receiver. + * + * @param receiver an Object to receive the method invocation + * @return a {@link BoundMethod} for this method and the receiver + * @throws IllegalStateException if the method is static + * @throws IllegalArgumentException if the receiver's class is incompatible + */ + public BoundMethod bind(Object receiver) { + Preconditions.checkState( + !isStatic(), "Cannot bind static method %s", method.toGenericString()); + Preconditions.checkArgument( + method.getDeclaringClass().isAssignableFrom(receiver.getClass()), + "Cannot bind %s to instance of %s", + method.toGenericString(), + receiver.getClass()); + + return new BoundMethod(this, receiver); + } + + /** Returns whether the method is a static method. */ + public boolean isStatic() { + return Modifier.isStatic(method.getModifiers()); + } + + /** Returns whether the method is a noop. */ + public boolean isNoop() { + return this == NOOP; + } + + /** + * Returns this method as a StaticMethod. + * + * @return a {@link StaticMethod} for this method + * @throws IllegalStateException if the method is not static + */ + public StaticMethod asStatic() { + Preconditions.checkState(isStatic(), "Method is not static"); + return new StaticMethod(this); + } + + @Override + public String toString() { + return "DynMethods.UnboundMethod(name=" + name + " method=" + method.toGenericString() + ")"; + } + + /** Singleton {@link UnboundMethod}, performs no operation and returns null. */ + private static final UnboundMethod NOOP = + new UnboundMethod(null, "NOOP") { + @Override + public R invokeChecked(Object target, Object... args) throws Exception { + return null; + } + + @Override + public BoundMethod bind(Object receiver) { + return new BoundMethod(this, receiver); + } + + @Override + public StaticMethod asStatic() { + return new StaticMethod(this); + } + + @Override + public boolean isStatic() { + return true; + } + + @Override + public String toString() { + return "DynMethods.UnboundMethod(NOOP)"; + } + }; + } + + public static class BoundMethod { + private final UnboundMethod method; + private final Object receiver; + + private BoundMethod(UnboundMethod method, Object receiver) { + this.method = method; + this.receiver = receiver; + } + + public R invokeChecked(Object... args) throws Exception { + return method.invokeChecked(receiver, args); + } + + public R invoke(Object... args) { + return method.invoke(receiver, args); + } + } + + public static class StaticMethod { + private final UnboundMethod method; + + private StaticMethod(UnboundMethod method) { + this.method = method; + } + + public R invokeChecked(Object... args) throws Exception { + return method.invokeChecked(null, args); + } + + public R invoke(Object... args) { + return method.invoke(null, args); + } + } + + /** + * Constructs a new builder for calling methods dynamically. + * + * @param methodName name of the method the builder will locate + * @return a Builder for finding a method + */ + public static Builder builder(String methodName) { + return new Builder(methodName); + } + + public static class Builder { + private final String name; + private UnboundMethod method = null; + + public Builder(String methodName) { + this.name = methodName; + } + + /** + * Set the {@link ClassLoader} used to lookup classes by name. + * + *

If not set, the current thread's ClassLoader is used. + * + * @param newLoader a ClassLoader + * @return this Builder for method chaining + */ + public Builder loader(ClassLoader newLoader) { + return this; + } + + /** + * If no implementation has been found, adds a NOOP method. + * + *

Note: calls to impl will not match after this method is called! + * + * @return this Builder for method chaining + */ + public Builder orNoop() { + if (method == null) { + this.method = UnboundMethod.NOOP; + } + return this; + } + + /** + * Checks for an implementation, first finding the given class by name. + * + * @param className name of a class + * @param methodName name of a method (different from constructor) + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder impl(String className, String methodName, Class... argClasses) { + // don't do any work if an implementation has been found + if (method != null) { + return this; + } + + try { + Class targetClass = Class.forName(className); + impl(targetClass, methodName, argClasses); + } catch (ClassNotFoundException e) { + // not the right implementation + } + return this; + } + + /** + * Checks for an implementation, first finding the given class by name. + * + *

The name passed to the constructor is the method name used. + * + * @param className name of a class + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder impl(String className, Class... argClasses) { + impl(className, name, argClasses); + return this; + } + + /** + * Checks for a method implementation. + * + * @param targetClass a class instance + * @param methodName name of a method (different from constructor) + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder impl(Class targetClass, String methodName, Class... argClasses) { + // don't do any work if an implementation has been found + if (method != null) { + return this; + } + + try { + this.method = new UnboundMethod(targetClass.getMethod(methodName, argClasses), name); + } catch (NoSuchMethodException e) { + // not the right implementation + } + return this; + } + + /** + * Checks for a method implementation. + * + *

The name passed to the constructor is the method name used. + * + * @param targetClass a class instance + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder impl(Class targetClass, Class... argClasses) { + impl(targetClass, name, argClasses); + return this; + } + + public Builder ctorImpl(Class targetClass, Class... argClasses) { + // don't do any work if an implementation has been found + if (method != null) { + return this; + } + + try { + this.method = new DynConstructors.Builder().impl(targetClass, argClasses).buildChecked(); + } catch (NoSuchMethodException e) { + // not the right implementation + } + return this; + } + + public Builder ctorImpl(String className, Class... argClasses) { + // don't do any work if an implementation has been found + if (method != null) { + return this; + } + + try { + this.method = new DynConstructors.Builder().impl(className, argClasses).buildChecked(); + } catch (NoSuchMethodException e) { + // not the right implementation + } + return this; + } + + /** + * Checks for an implementation, first finding the given class by name. + * + * @param className name of a class + * @param methodName name of a method (different from constructor) + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder hiddenImpl(String className, String methodName, Class... argClasses) { + // don't do any work if an implementation has been found + if (method != null) { + return this; + } + + try { + Class targetClass = Class.forName(className ); + hiddenImpl(targetClass, methodName, argClasses); + } catch (ClassNotFoundException e) { + // not the right implementation + } + return this; + } + + /** + * Checks for an implementation, first finding the given class by name. + * + *

The name passed to the constructor is the method name used. + * + * @param className name of a class + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder hiddenImpl(String className, Class... argClasses) { + hiddenImpl(className, name, argClasses); + return this; + } + + /** + * Checks for a method implementation. + * + * @param targetClass a class instance + * @param methodName name of a method (different from constructor) + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder hiddenImpl(Class targetClass, String methodName, Class... argClasses) { + // don't do any work if an implementation has been found + if (method != null) { + return this; + } + + try { + Method hidden = targetClass.getDeclaredMethod(methodName, argClasses); + AccessController.doPrivileged(new MakeAccessible(hidden)); + this.method = new UnboundMethod(hidden, name); + } catch (SecurityException | NoSuchMethodException e) { + // unusable or not the right implementation + } + return this; + } + + /** + * Checks for a method implementation. + * + *

The name passed to the constructor is the method name used. + * + * @param targetClass a class instance + * @param argClasses argument classes for the method + * @return this Builder for method chaining + * @see Class#forName(String) + * @see Class#getMethod(String, Class[]) + */ + public Builder hiddenImpl(Class targetClass, Class... argClasses) { + hiddenImpl(targetClass, name, argClasses); + return this; + } + + /** + * Returns the first valid implementation as a UnboundMethod or throws a RuntimeError if there + * is none. + * + * @return a {@link UnboundMethod} with a valid implementation + * @throws RuntimeException if no implementation was found + */ + public UnboundMethod build() { + if (method != null) { + return method; + } else { + throw new RuntimeException("Cannot find method: " + name); + } + } + + /** + * Returns the first valid implementation as a BoundMethod or throws a RuntimeError if there is + * none. + * + * @param receiver an Object to receive the method invocation + * @return a {@link BoundMethod} with a valid implementation and receiver + * @throws IllegalStateException if the method is static + * @throws IllegalArgumentException if the receiver's class is incompatible + * @throws RuntimeException if no implementation was found + */ + public BoundMethod build(Object receiver) { + return build().bind(receiver); + } + + /** + * Returns the first valid implementation as a UnboundMethod or throws a NoSuchMethodException + * if there is none. + * + * @return a {@link UnboundMethod} with a valid implementation + * @throws NoSuchMethodException if no implementation was found + */ + public UnboundMethod buildChecked() throws NoSuchMethodException { + if (method != null) { + return method; + } else { + throw new NoSuchMethodException("Cannot find method: " + name); + } + } + + /** + * Returns the first valid implementation as a BoundMethod or throws a NoSuchMethodException if + * there is none. + * + * @param receiver an Object to receive the method invocation + * @return a {@link BoundMethod} with a valid implementation and receiver + * @throws IllegalStateException if the method is static + * @throws IllegalArgumentException if the receiver's class is incompatible + * @throws NoSuchMethodException if no implementation was found + */ + public BoundMethod buildChecked(Object receiver) throws NoSuchMethodException { + return buildChecked().bind(receiver); + } + + /** + * Returns the first valid implementation as a StaticMethod or throws a NoSuchMethodException if + * there is none. + * + * @return a {@link StaticMethod} with a valid implementation + * @throws IllegalStateException if the method is not static + * @throws NoSuchMethodException if no implementation was found + */ + public StaticMethod buildStaticChecked() throws NoSuchMethodException { + return buildChecked().asStatic(); + } + + /** + * Returns the first valid implementation as a StaticMethod or throws a RuntimeException if + * there is none. + * + * @return a {@link StaticMethod} with a valid implementation + * @throws IllegalStateException if the method is not static + * @throws RuntimeException if no implementation was found + */ + public StaticMethod buildStatic() { + return build().asStatic(); + } + } + + private static class MakeAccessible implements PrivilegedAction { + private Method hidden; + + MakeAccessible(Method hidden) { + this.hidden = hidden; + } + + @Override + public Void run() { + hidden.setAccessible(true); + return null; + } + } +} diff --git a/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/README.md b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/README.md new file mode 100644 index 0000000000000..e817f66e2d93e --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/main/java/org/apache/iceberg/common/README.md @@ -0,0 +1,6 @@ +# Why we need this package? + +In this package we have override the `iceberg-common` package, since in original `iceberg-common` it uses `Thread.getContextClassLoader` to load classes dynamically. +While this works well in most cases, it will fail when invoked by jni, since by default jni threads was passed bootstrap class loader, and `Thread.getContextClassLoader` +will inherit parent thread's class loader. That's to say, all threads created by jni will use bootstrap class loader. While we can use `Thread.setContextClassLoader` to it system class loader +manually, but it's not possible in all cases since iceberg used thread pools internally, which can't be hooked by us. \ No newline at end of file diff --git a/java/connector-node/risingwave-sink-iceberg/src/test/java/com/risingwave/connector/catalog/JniCatalogWrapperTest.java b/java/connector-node/risingwave-sink-iceberg/src/test/java/com/risingwave/connector/catalog/JniCatalogWrapperTest.java new file mode 100644 index 0000000000000..0edf0bce9a7d6 --- /dev/null +++ b/java/connector-node/risingwave-sink-iceberg/src/test/java/com/risingwave/connector/catalog/JniCatalogWrapperTest.java @@ -0,0 +1,44 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package com.risingwave.connector.catalog; + +import org.junit.Ignore; +import org.junit.Test; + +@Ignore +public class JniCatalogWrapperTest { + @Test + public void testJdbc() throws Exception { + System.setProperty("aws.region", "us-east-1"); + JniCatalogWrapper catalog = + JniCatalogWrapper.create( + "demo", + "org.apache.iceberg.jdbc.JdbcCatalog", + new String[] { + "uri", "jdbc:postgresql://172.17.0.3:5432/iceberg", + "jdbc.user", "admin", + "jdbc.password", "123456", + "warehouse", "s3://icebergdata/demo", + "io-impl", "org.apache.iceberg.aws.s3.S3FileIO", + "s3.endpoint", "http://172.17.0.2:9301", + "s3.region", "us-east-1", + "s3.path-style-access", "true", + "s3.access-key-id", "hummockadmin", + "s3.secret-access-key", "hummockadmin", + }); + + System.out.println(catalog.loadTable("s1.t1")); + } +} diff --git a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java index 52213b224c151..dfdc01f4a36ed 100644 --- a/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java +++ b/java/connector-node/risingwave-sink-jdbc/src/main/java/com/risingwave/connector/JDBCSink.java @@ -64,7 +64,11 @@ public JDBCSink(JDBCSinkConfig config, TableSchema tableSchema) { var columnName = tableSchema.getColumnNames()[columnIdx]; columnSqlTypes[columnIdx] = columnTypeMapping.get(columnName); } - LOG.info("columnSqlTypes: {}", Arrays.toString(columnSqlTypes)); + LOG.info( + "schema = {}, table = {}: columnSqlTypes = {}", + config.getSchemaName(), + config.getTableName(), + Arrays.toString(columnSqlTypes)); if (factory.isPresent()) { this.jdbcDialect = factory.get().create(columnSqlTypes); @@ -105,7 +109,11 @@ private static Map getColumnTypeMapping( String.format(ERROR_REPORT_TEMPLATE, e.getSQLState(), e.getMessage())) .asRuntimeException(); } - LOG.info("detected column type mapping {}", columnTypeMap); + LOG.info( + "schema = {}, table = {}: detected column type mapping = {}", + schemaName, + tableName, + columnTypeMap); return columnTypeMap; } @@ -123,7 +131,11 @@ private static List getPkColumnNames( String.format(ERROR_REPORT_TEMPLATE, e.getSQLState(), e.getMessage())) .asRuntimeException(); } - LOG.info("detected pk column {}", pkColumnNames); + LOG.info( + "schema = {}, table = {}: detected pk column = {}", + schemaName, + tableName, + pkColumnNames); return pkColumnNames; } diff --git a/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-http-sink/src/main/java/com/risingwave/mock/flink/http/HttpFlinkMockSinkFactory.java b/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-http-sink/src/main/java/com/risingwave/mock/flink/http/HttpFlinkMockSinkFactory.java index a969dddd620f7..d316eeae74bed 100644 --- a/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-http-sink/src/main/java/com/risingwave/mock/flink/http/HttpFlinkMockSinkFactory.java +++ b/java/connector-node/risingwave-sink-mock-flink/risingwave-sink-mock-flink-http-sink/src/main/java/com/risingwave/mock/flink/http/HttpFlinkMockSinkFactory.java @@ -26,6 +26,8 @@ /** * The `FlinkMockSinkFactory` implementation of the http sink is responsible for creating the http * counterpart of the `DynamicTableSinkFactory`. And `validate` don't need to do anything. + * + *

This feature is depended on https://github.com/getindata/flink-http-connector */ public class HttpFlinkMockSinkFactory implements FlinkMockSinkFactory { @Override diff --git a/java/connector-node/risingwave-source-cdc/pom.xml b/java/connector-node/risingwave-source-cdc/pom.xml index 683f472a65b14..fd1dccd3f092f 100644 --- a/java/connector-node/risingwave-source-cdc/pom.xml +++ b/java/connector-node/risingwave-source-cdc/pom.xml @@ -43,6 +43,10 @@ io.debezium debezium-connector-mysql + + io.debezium + debezium-connector-mongodb + @@ -58,5 +62,9 @@ org.postgresql postgresql + + org.mongodb + mongodb-driver-sync + diff --git a/java/connector-node/risingwave-source-test/pom.xml b/java/connector-node/risingwave-source-test/pom.xml index 46956b642f15e..16bc9e7c75427 100644 --- a/java/connector-node/risingwave-source-test/pom.xml +++ b/java/connector-node/risingwave-source-test/pom.xml @@ -19,21 +19,6 @@ connector-api test - - com.risingwave - s3-common - test - - - org.slf4j - slf4j-log4j12 - - - org.slf4j - slf4j-reload4j - - - io.grpc grpc-protobuf @@ -46,11 +31,19 @@ org.apache.logging.log4j log4j-core + + org.apache.logging.log4j + log4j-slf4j2-impl + junit junit test + + io.debezium + debezium-connector-mongodb + org.assertj assertj-core @@ -75,6 +68,16 @@ org.testcontainers postgresql + + org.testcontainers + mongodb + + + + org.mongodb + mongodb-driver-sync + 4.11.1 + com.fasterxml.jackson.core jackson-databind diff --git a/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/SourceTestClient.java b/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/SourceTestClient.java index 55792b84eb809..58c72c688dfef 100644 --- a/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/SourceTestClient.java +++ b/java/connector-node/risingwave-source-test/src/test/java/com/risingwave/connector/source/SourceTestClient.java @@ -29,6 +29,7 @@ import java.sql.SQLException; import java.sql.Statement; import java.util.Iterator; +import java.util.Map; import java.util.Properties; import java.util.Random; import java.util.UUID; @@ -152,6 +153,25 @@ protected Iterator getEventStreamS return responses; } + public Iterator getEventStream( + ConnectorServiceProto.SourceType sourceType, + long sourceId, + Map properties) { + ConnectorServiceProto.GetEventStreamRequest req = + ConnectorServiceProto.GetEventStreamRequest.newBuilder() + .setSourceId(sourceId) + .setSourceType(sourceType) + .putAllProperties(properties) + .build(); + Iterator responses = null; + try { + responses = blockingStub.getEventStream(req); + } catch (StatusRuntimeException e) { + fail("RPC failed: {}", e.getStatus()); + } + return responses; + } + // generates an orders.tbl in class path using random data // if file does not contain 10000 lines static void genOrdersTable(int numRows) { diff --git a/java/connector-node/risingwave-source-test/src/test/resources/log4j2.properties b/java/connector-node/risingwave-source-test/src/test/resources/log4j2.properties index d46bd5f609266..856287e831da8 100644 --- a/java/connector-node/risingwave-source-test/src/test/resources/log4j2.properties +++ b/java/connector-node/risingwave-source-test/src/test/resources/log4j2.properties @@ -1,4 +1,4 @@ -rootLogger.level=ERROR +rootLogger.level=INFO # declare the appender to use appenders=console # appender properties diff --git a/java/connector-node/tracing/src/main/java/com/risingwave/tracing/TracingSlf4jAdapter.java b/java/connector-node/tracing/src/main/java/com/risingwave/tracing/TracingSlf4jAdapter.java index d5592285896be..5ccef41e9ac13 100644 --- a/java/connector-node/tracing/src/main/java/com/risingwave/tracing/TracingSlf4jAdapter.java +++ b/java/connector-node/tracing/src/main/java/com/risingwave/tracing/TracingSlf4jAdapter.java @@ -52,8 +52,12 @@ private void logIfEnabled(int level, String format, Object arg1, Object arg2) { private void logIfEnabled(int level, String format, Object... arguments) { if (TracingSlf4jImpl.isEnabled(level)) { - TracingSlf4jImpl.event( - name, level, new ParameterizedMessage(format, arguments).getFormattedMessage()); + var pm = new ParameterizedMessage(format, arguments); + if (null != pm.getThrowable()) { + logIfEnabled(level, pm.getFormattedMessage(), pm.getThrowable()); + } else { + TracingSlf4jImpl.event(name, level, pm.getFormattedMessage()); + } } } diff --git a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java index fbd952cc68e64..a16acda73e7fd 100644 --- a/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java +++ b/java/java-binding/src/main/java/com/risingwave/java/binding/Binding.java @@ -90,6 +90,8 @@ public static native void tracingSlf4jEvent( public static native boolean sendCdcSourceMsgToChannel(long channelPtr, byte[] msg); + public static native boolean sendCdcSourceErrorToChannel(long channelPtr, String errorMsg); + public static native com.risingwave.java.binding.JniSinkWriterStreamRequest recvSinkWriterRequestFromChannel(long channelPtr); diff --git a/java/pom.xml b/java/pom.xml index 5f168c48bd9ef..2cf4986f82ed4 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -69,7 +69,7 @@ 1.53.0 2.10 0.1.0-SNAPSHOT - 2.27.1 + 2.43.0 2.20.0 2.0.9 1.5.0 @@ -84,6 +84,12 @@ 4.15.0 1.18.0 1.17.6 + 42.5.1 + 8.0.28 + 4.11.1 + 3.45.0.0 + 2.21.42 + 3.1.3 @@ -95,7 +101,7 @@ org.apache.logging.log4j - log4j-slf4j-impl + log4j-slf4j2-impl ${log4j.version} @@ -161,15 +167,25 @@ debezium-connector-mysql ${debezium.version} + + io.debezium + debezium-connector-mongodb + ${debezium.version} + org.postgresql postgresql - 42.5.1 + ${postgresql.version} mysql mysql-connector-java - 8.0.28 + ${mysql.connector.java.version} + + + org.mongodb + mongodb-driver-sync + ${mongodb.driver.sync.version} org.elasticsearch @@ -317,13 +333,58 @@ org.apache.httpcomponents httpclient - 4.5.10 + 4.5.13 io.prometheus simpleclient_httpserver 0.5.0 + + org.xerial + sqlite-jdbc + ${sqlite.version} + + + software.amazon.awssdk + s3 + ${aws.version} + + + software.amazon.awssdk + sts + ${aws.version} + + + software.amazon.awssdk + apache-client + ${aws.version} + + + org.apache.hadoop + hadoop-common + ${hadoop.version} + + + org.apache.hive + hive-metastore + ${hive.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-common + ${hadoop.version} + + + org.apache.hadoop + hadoop-mapreduce-client-jobclient + ${hadoop.version} + org.apache.spark spark-sql_2.12 @@ -348,6 +409,12 @@ ${testcontainers.version} test + + org.testcontainers + mongodb + ${testcontainers.version} + test + @@ -381,6 +448,7 @@ /tools/maven/checkstyle.xml true true + **/org/apache/iceberg/common/* @@ -391,11 +459,14 @@ - 1.7 + 1.20.0 + + **/org/apache/iceberg/common/** + diff --git a/java/tools/maven/checkstyle.xml b/java/tools/maven/checkstyle.xml index f6816332723c6..b46efef4d313f 100644 --- a/java/tools/maven/checkstyle.xml +++ b/java/tools/maven/checkstyle.xml @@ -112,27 +112,6 @@ This file is based on the checkstyle file of Apache Beam. - - - - - - - - - - - - - - - - - diff --git a/lints/Cargo.lock b/lints/Cargo.lock index daac2a9301ddb..b8fd465fecd7b 100644 --- a/lints/Cargo.lock +++ b/lints/Cargo.lock @@ -111,16 +111,25 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clippy_config" +version = "0.1.77" +source = "git+https://github.com/rust-lang/rust-clippy?rev=6fd0258e45105161b7e759a22e7350958e5cb0b1#6fd0258e45105161b7e759a22e7350958e5cb0b1" +dependencies = [ + "rustc-semver", + "serde", + "toml 0.7.8", +] + [[package]] name = "clippy_utils" -version = "0.1.75" -source = "git+https://github.com/rust-lang/rust-clippy?rev=a585cda701581a16894858dc088eacd5a02fc78b#a585cda701581a16894858dc088eacd5a02fc78b" +version = "0.1.77" +source = "git+https://github.com/rust-lang/rust-clippy?rev=6fd0258e45105161b7e759a22e7350958e5cb0b1#6fd0258e45105161b7e759a22e7350958e5cb0b1" dependencies = [ "arrayvec", - "if_chain", - "itertools 0.10.5", + "clippy_config", + "itertools 0.11.0", "rustc-semver", - "serde", ] [[package]] @@ -224,9 +233,9 @@ dependencies = [ [[package]] name = "dylint" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683b9feb84cd2620b4b75119b7e914ac132dbd9e523f9a98821f3b3a7e355053" +checksum = "71fdb7b800ab13925402f0048ed0911068db2e5ba6168dd93962269d4f39541d" dependencies = [ "ansi_term", "anyhow", @@ -245,9 +254,9 @@ dependencies = [ [[package]] name = "dylint_internal" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ede982d9261f23a19b92ed7dc4ddeefc8328fc21c88e2c79ffd6e071c7972be" +checksum = "5154dada2bee2a69f75f54eae57479f56f93ca1db80725a1d82cdb5fe231ef73" dependencies = [ "ansi_term", "anyhow", @@ -257,15 +266,16 @@ dependencies = [ "if_chain", "is-terminal", "log", + "once_cell", "rust-embed", "sedregex", ] [[package]] name = "dylint_linting" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee7f2f02100bafd2f02c5fcc4ca981adca28397485bf3393ad0a1e8e46ec583b" +checksum = "d203baeb8770847314632f652e0e62dd7fec6a21102a116472eec0d6931f5dd9" dependencies = [ "cargo_metadata", "dylint_internal", @@ -273,14 +283,14 @@ dependencies = [ "rustversion", "serde", "thiserror", - "toml", + "toml 0.8.8", ] [[package]] name = "dylint_testing" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "964c9965a990e1ab44a862cb9823772603fe4d82b777b1190fe2d4d62ed8a885" +checksum = "1208b1f2c40fc2f3c3fa0d5631efbc7a95721d619410dc2da5b0496810d6a941" dependencies = [ "anyhow", "cargo_metadata", @@ -388,11 +398,11 @@ dependencies = [ [[package]] name = "git2" -version = "0.17.2" +version = "0.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b989d6a7ca95a362cf2cfc5ad688b3a467be1f87e480b8dad07fee8c79b0044" +checksum = "fbf97ba92db08df386e10c8ede66a2a0369bd277090afd8710e19e38de9ec0cd" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.4.1", "libc", "libgit2-sys", "log", @@ -480,9 +490,9 @@ dependencies = [ [[package]] name = "itertools" -version = "0.10.5" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" dependencies = [ "either", ] @@ -519,15 +529,15 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.150" +version = "0.2.152" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89d92a4743f9a61002fae18374ed11e7973f530cb3a3255fb354818118b2203c" +checksum = "13e3bf6590cbc649f4d1a3eefc9d5d6eb746f5200ffb04e5e142700b8faa56e7" [[package]] name = "libgit2-sys" -version = "0.15.2+1.6.4" +version = "0.16.2+1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a80df2e11fb4a61f4ba2ab42dbe7f74468da143f1a75c74e11dee7c813f694fa" +checksum = "ee4126d8b4ee5c9d9ea891dd875cfdc1e9d0950437179104b183d7d8a74d24e8" dependencies = [ "cc", "libc", @@ -578,18 +588,20 @@ dependencies = [ name = "lints" version = "0.1.0" dependencies = [ + "anyhow", "clippy_utils", "dylint_linting", "dylint_testing", "itertools 0.12.0", + "thiserror-ext", "tracing", ] [[package]] name = "linux-raw-sys" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "969488b55f8ac402214f3f5fd243ebb7206cf82de60d3172994707a4bcc2b829" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "log" @@ -624,9 +636,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "openssl-probe" @@ -754,9 +766,9 @@ checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "rust-embed" -version = "8.0.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b1e7d90385b59f0a6bf3d3b757f3ca4ece2048265d70db20a2016043d4509a40" +checksum = "a82c0bbc10308ed323529fd3c1dce8badda635aa319a5ff0e6466f33b8101e3f" dependencies = [ "rust-embed-impl", "rust-embed-utils", @@ -765,9 +777,9 @@ dependencies = [ [[package]] name = "rust-embed-impl" -version = "8.0.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c3d8c6fd84090ae348e63a84336b112b5c3918b3bf0493a581f7bd8ee623c29" +checksum = "6227c01b1783cdfee1bcf844eb44594cd16ec71c35305bf1c9fb5aade2735e16" dependencies = [ "proc-macro2", "quote", @@ -778,9 +790,9 @@ dependencies = [ [[package]] name = "rust-embed-utils" -version = "8.0.0" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "873feff8cb7bf86fdf0a71bb21c95159f4e4a37dd7a4bd1855a940909b583ada" +checksum = "8cb0a25bfbb2d4b4402179c2cf030387d9990857ce08a32592c6238db9fa8665" dependencies = [ "globset", "sha2", @@ -807,15 +819,15 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.25" +version = "0.38.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc99bc2d4f1fed22595588a013687477aedf3cdcfb26558c559edb67b4d9b22e" +checksum = "322394588aaf33c24007e8bb3238ee3e4c5c09c084ab32bc73890b99ff326bca" dependencies = [ "bitflags 2.4.1", "errno", "libc", "linux-raw-sys", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -921,15 +933,15 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.8.1" +version = "3.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef1adac450ad7f4b3c28589471ade84f25f731a7a0fe30d71dfa9f60fd808e5" +checksum = "01ce4141aa927a6d1bd34a041795abd0db1cccba5d5f24b009f694bdf3a1f3fa" dependencies = [ "cfg-if", "fastrand", "redox_syscall 0.4.1", "rustix", - "windows-sys 0.48.0", + "windows-sys 0.52.0", ] [[package]] @@ -974,6 +986,28 @@ dependencies = [ "thiserror-impl", ] +[[package]] +name = "thiserror-ext" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368fdd73eb5f0df62797d1a5d1dca41ca4137a62f250980787ffe33e530b5d8b" +dependencies = [ + "thiserror", + "thiserror-ext-derive", +] + +[[package]] +name = "thiserror-ext-derive" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "efc1991800eaf856df8d097c909b487fdee13db47ffa293136c91cde6de4abee" +dependencies = [ + "either", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thiserror-impl" version = "1.0.50" @@ -1000,6 +1034,18 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "toml" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd79e69d3b627db300ff956027cc6c3798cef26d22526befdfcd12feeb6d2257" +dependencies = [ + "serde", + "serde_spanned", + "toml_datetime", + "toml_edit 0.19.15", +] + [[package]] name = "toml" version = "0.8.8" @@ -1009,7 +1055,7 @@ dependencies = [ "serde", "serde_spanned", "toml_datetime", - "toml_edit", + "toml_edit 0.21.0", ] [[package]] @@ -1021,6 +1067,19 @@ dependencies = [ "serde", ] +[[package]] +name = "toml_edit" +version = "0.19.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421" +dependencies = [ + "indexmap", + "serde", + "serde_spanned", + "toml_datetime", + "winnow", +] + [[package]] name = "toml_edit" version = "0.21.0" diff --git a/lints/Cargo.toml b/lints/Cargo.toml index 9b0cbdcfc0522..14bd09faca184 100644 --- a/lints/Cargo.toml +++ b/lints/Cargo.toml @@ -11,15 +11,19 @@ crate-type = ["cdylib"] name = "format_error" path = "ui/format_error.rs" +# See `README.md` before bumping the version. +# Remember to update the version in `ci/Dockerfile` as well. [dependencies] -clippy_utils = { git = "https://github.com/rust-lang/rust-clippy", rev = "e4c626dd9a17a23270bf8e7158e59cf2b9c04840" } # should match the toolchain version (rustc -vV) -dylint_linting = "2.5.0" +clippy_utils = { git = "https://github.com/rust-lang/rust-clippy", rev = "6fd0258e45105161b7e759a22e7350958e5cb0b1" } +dylint_linting = "2.6.0" itertools = "0.12" [dev-dependencies] -dylint_testing = "2.5.0" +dylint_testing = "2.6.0" # UI test dependencies +anyhow = "1" +thiserror-ext = "0.1" tracing = "0.1" [package.metadata.rust-analyzer] diff --git a/lints/README.md b/lints/README.md index 7b73f84296ade..5007474227ab3 100644 --- a/lints/README.md +++ b/lints/README.md @@ -4,10 +4,18 @@ Custom lints for RisingWave to enforce code style and best practices, empowered See [cargo dylint](https://github.com/trailofbits/dylint) for more information. +## Install `cargo-dylint` + +```bash +cargo install cargo-dylint dylint-link +``` + ## Run lints To run all lints, run `cargo dylint --all` in the root of the repository. +If you find there are some compile errors, try updating the `cargo-dylint` binary by installing it again. + ## Add new lints To add a new lint, add a new file in the `src` directory to declare the lint, then register it in `fn register_lints(..)` in `lib.rs`. @@ -19,3 +27,16 @@ To test a lint, create a new file in the `ui` directory and add it as an `exampl ## VS Code integration Duplicate `.vscode/settings.json.example` to `.vscode/settings.json` to enable rust-analyzer integration for developing lints. + +## Bump toolchain + +The version of the toolchain is specified in `rust-toolchain` file under current directory. +It does not have to be exactly the same as the one used to build RisingWave, but it should be close enough to avoid compile errors. + +The information below can be helpful in finding the appropriate version to bump to. + +- The toolchain used by the latest version of `cargo-dylint`: https://github.com/trailofbits/dylint/blob/master/internal/template/rust-toolchain +- The toolchain used by the latest version of `clippy`: https://github.com/rust-lang/rust-clippy/blob/master/rust-toolchain +- The hash of the latest commit in `rust-lang/rust-clippy` repo for the dependency `clippy-utils`. + +Run the lints after bumping the toolchain to verify it works. diff --git a/lints/rust-toolchain b/lints/rust-toolchain index 975abccbffd50..ea1f0e928e5c3 100644 --- a/lints/rust-toolchain +++ b/lints/rust-toolchain @@ -1,3 +1,5 @@ +# See `README.md` before bumping the version. + [toolchain] -channel = "nightly-2023-12-26" # should be identical to the root one +channel = "nightly-2024-01-11" components = ["llvm-tools-preview", "rustc-dev"] diff --git a/lints/src/format_error.rs b/lints/src/format_error.rs index 46770e2b76aa1..5d95921e48f38 100644 --- a/lints/src/format_error.rs +++ b/lints/src/format_error.rs @@ -16,8 +16,10 @@ use clippy_utils::diagnostics::span_lint_and_help; use clippy_utils::macros::{ find_format_arg_expr, find_format_args, is_format_macro, macro_backtrace, }; -use clippy_utils::ty::implements_trait; -use clippy_utils::{is_in_cfg_test, is_in_test_function, is_trait_method, match_function_call}; +use clippy_utils::ty::{implements_trait, match_type}; +use clippy_utils::{ + is_in_cfg_test, is_in_test_function, is_trait_method, match_def_path, match_function_call, +}; use rustc_ast::FormatArgsPiece; use rustc_hir::{Expr, ExprKind}; use rustc_lint::{LateContext, LateLintPass}; @@ -60,6 +62,10 @@ impl_lint_pass!(FormatError => [FORMAT_ERROR]); const TRACING_FIELD_DEBUG: [&str; 3] = ["tracing_core", "field", "debug"]; const TRACING_FIELD_DISPLAY: [&str; 3] = ["tracing_core", "field", "display"]; +const TRACING_MACROS_EVENT: [&str; 3] = ["tracing", "macros", "event"]; +const ANYHOW_MACROS_ANYHOW: [&str; 3] = ["anyhow", "macros", "anyhow"]; +const ANYHOW_ERROR: [&str; 2] = ["anyhow", "Error"]; +const THISERROR_EXT_REPORT_REPORT: [&str; 3] = ["thiserror_ext", "report", "Report"]; impl<'tcx> LateLintPass<'tcx> for FormatError { fn check_expr(&mut self, cx: &LateContext<'tcx>, expr: &'tcx Expr<'_>) { @@ -73,10 +79,15 @@ impl<'tcx> LateLintPass<'tcx> for FormatError { .or_else(|| match_function_call(cx, expr, &TRACING_FIELD_DISPLAY)) && let [arg_expr, ..] = args { - check_fmt_arg(cx, arg_expr); + check_fmt_arg_in_tracing_event(cx, arg_expr); } - // `{}`, `{:?}` in format macros. + // Indirect `{}`, `{:?}` from other macros. + let in_tracing_event_macro = macro_backtrace(expr.span) + .any(|macro_call| match_def_path(cx, macro_call.def_id, &TRACING_MACROS_EVENT)); + let in_anyhow_macro = macro_backtrace(expr.span) + .any(|macro_call| match_def_path(cx, macro_call.def_id, &ANYHOW_MACROS_ANYHOW)); + for macro_call in macro_backtrace(expr.span) { if is_format_macro(cx, macro_call.def_id) && let Some(format_args) = find_format_args(cx, expr, macro_call.expn) @@ -87,7 +98,17 @@ impl<'tcx> LateLintPass<'tcx> for FormatError { && let Some(arg) = format_args.arguments.all_args().get(index) && let Ok(arg_expr) = find_format_arg_expr(expr, arg) { - check_fmt_arg(cx, arg_expr); + if in_tracing_event_macro { + check_fmt_arg_in_tracing_event(cx, arg_expr); + } else if in_anyhow_macro { + if format_args.template.len() == 1 { + check_fmt_arg_in_anyhow_error(cx, arg_expr); + } else { + check_fmt_arg_in_anyhow_context(cx, arg_expr); + } + } else { + check_fmt_arg(cx, arg_expr); + } } } } @@ -103,14 +124,54 @@ impl<'tcx> LateLintPass<'tcx> for FormatError { } fn check_fmt_arg(cx: &LateContext<'_>, arg_expr: &Expr<'_>) { - check_arg( + check_fmt_arg_with_help( cx, arg_expr, - arg_expr.span, "consider importing `thiserror_ext::AsReport` and using `.as_report()` instead", + ) +} + +fn check_fmt_arg_in_tracing_event(cx: &LateContext<'_>, arg_expr: &Expr<'_>) { + // TODO: replace `` with the actual code snippet. + check_fmt_arg_with_help( + cx, + arg_expr, + "consider importing `thiserror_ext::AsReport` and recording the error as a field \ + with `error = %.as_report()` instead", ); } +fn check_fmt_arg_in_anyhow_error(cx: &LateContext<'_>, arg_expr: &Expr<'_>) { + check_fmt_arg_with_help( + cx, + arg_expr, + ( + "consider directly wrapping the error with `anyhow::anyhow!(..)` instead of formatting it", + "consider removing the redundant wrapping of `anyhow::anyhow!(..)`", + "consider directly wrapping the error with `anyhow::anyhow!(..)` instead of formatting its report", + ), + ); +} + +fn check_fmt_arg_in_anyhow_context(cx: &LateContext<'_>, arg_expr: &Expr<'_>) { + check_fmt_arg_with_help( + cx, + arg_expr, + ( + "consider using `anyhow::Context::(with_)context` to \ + attach additional message to the error and make it an error source instead", + "consider using `.context(..)` to \ + attach additional message to the error and make it an error source instead", + "consider using `anyhow::Context::(with_)context` to \ + attach additional message to the error and make it an error source instead", + ), + ); +} + +fn check_fmt_arg_with_help(cx: &LateContext<'_>, arg_expr: &Expr<'_>, help: impl Help) { + check_arg(cx, arg_expr, arg_expr.span, help); +} + fn check_to_string_call(cx: &LateContext<'_>, receiver: &Expr<'_>, to_string_span: Span) { check_arg( cx, @@ -120,27 +181,69 @@ fn check_to_string_call(cx: &LateContext<'_>, receiver: &Expr<'_>, to_string_spa ); } -fn check_arg(cx: &LateContext<'_>, arg_expr: &Expr<'_>, span: Span, help: &str) { +fn check_arg(cx: &LateContext<'_>, arg_expr: &Expr<'_>, span: Span, help: impl Help) { let Some(error_trait_id) = cx.tcx.get_diagnostic_item(sym::Error) else { return; }; let ty = cx.typeck_results().expr_ty(arg_expr).peel_refs(); - if implements_trait(cx, ty, error_trait_id, &[]) { - if let Some(span) = core::iter::successors(Some(span), |s| s.parent_callsite()) - .find(|s| s.can_be_used_for_suggestions()) - { - // TODO: applicable suggestions - span_lint_and_help( - cx, - FORMAT_ERROR, - span, - "should not format error directly", - None, - help, - ); + let help = if implements_trait(cx, ty, error_trait_id, &[]) { + help.normal_help() + } else if match_type(cx, ty, &ANYHOW_ERROR) { + help.anyhow_help() + } else if match_type(cx, ty, &THISERROR_EXT_REPORT_REPORT) { + if let Some(help) = help.report_help() { + help + } else { + return; } + } else { + return; + }; + + if let Some(span) = core::iter::successors(Some(span), |s| s.parent_callsite()) + .find(|s| s.can_be_used_for_suggestions()) + { + // TODO: applicable suggestions + span_lint_and_help( + cx, + FORMAT_ERROR, + span, + "should not format error directly", + None, + help, + ); + } +} + +trait Help { + fn normal_help(&self) -> &str; + fn anyhow_help(&self) -> &str { + self.normal_help() + } + fn report_help(&self) -> Option<&str> { + None + } +} + +impl Help for &str { + fn normal_help(&self) -> &str { + self + } +} + +impl Help for (&str, &str, &str) { + fn normal_help(&self) -> &str { + self.0 + } + + fn anyhow_help(&self) -> &str { + self.1 + } + + fn report_help(&self) -> Option<&str> { + Some(self.2) } } diff --git a/lints/src/lib.rs b/lints/src/lib.rs index 264e0d47b62ae..df77538d3cf17 100644 --- a/lints/src/lib.rs +++ b/lints/src/lib.rs @@ -14,6 +14,7 @@ #![feature(rustc_private)] #![feature(let_chains)] +#![feature(lazy_cell)] #![warn(unused_extern_crates)] extern crate rustc_ast; @@ -32,10 +33,39 @@ dylint_linting::dylint_library!(); #[allow(clippy::no_mangle_with_rust_abi)] #[no_mangle] pub fn register_lints(_sess: &rustc_session::Session, lint_store: &mut rustc_lint::LintStore) { + // -- Begin lint registration -- + + // Preparation steps. lint_store.register_early_pass(|| { Box::::default() }); + // Actual lints. lint_store.register_lints(&[format_error::FORMAT_ERROR]); lint_store.register_late_pass(|_| Box::::default()); + + // -- End lint registration -- + + // Register lints into groups. + // Note: use `rw_` instead of `rw::` to avoid "error[E0602]: unknown lint tool: `rw`". + register_group(lint_store, "rw_all", |_| true); + register_group(lint_store, "rw_warnings", |l| { + l.default_level >= rustc_lint::Level::Warn + }); +} + +fn register_group( + lint_store: &mut rustc_lint::LintStore, + name: &'static str, + filter_predicate: impl Fn(&rustc_lint::Lint) -> bool, +) { + let lints = lint_store + .get_lints() + .iter() + .filter(|l| l.name.starts_with("rw::")) + .filter(|l| filter_predicate(l)) + .map(|l| rustc_lint::LintId::of(l)) + .collect(); + + lint_store.register_group(true, name, None, lints); } diff --git a/lints/ui/format_error.rs b/lints/ui/format_error.rs index 1cdc112bd93aa..22ea8c5f88e8b 100644 --- a/lints/ui/format_error.rs +++ b/lints/ui/format_error.rs @@ -48,4 +48,36 @@ fn main() { let _ = (err.clone()).to_string(); let _ = err.to_string().to_string(); let _ = (&&err).to_string(); + + use anyhow::anyhow; + + let _ = anyhow!("{}", err); + let _ = anyhow!("{:?}", err); + let _ = anyhow!("some error occurred: {}", err); + let _ = anyhow!("some error occurred: {:?}", err); + + // `anyhow::Error` does not implement `Error` trait, test the special path here. + let make_anyhow_err = || anyhow!("foobar"); + let anyhow_err = make_anyhow_err(); + + let _ = format!("{}", anyhow_err); + let _ = format!("{}", &anyhow_err); + let _ = format!("{}", &&anyhow_err); + let _ = format!("{}", Box::new(&anyhow_err)); // TODO: fail to lint + + tracing::field::display(&anyhow_err); + tracing::field::debug(make_anyhow_err()); + + let _ = anyhow_err.to_string(); + let _ = (&&anyhow_err).to_string(); + + let _ = anyhow!("{}", anyhow_err); + let _ = anyhow!("some error occurred: {:?}", anyhow_err); + + use thiserror_ext::AsReport; + + let _ = anyhow!("{}", err.as_report()); + let _ = anyhow!("some error occurred: {}", err.as_report()); + let _ = anyhow!("{:?}", anyhow_err.as_report()); + let _ = anyhow!("some error occurred: {:?}", anyhow_err.as_report()); } diff --git a/lints/ui/format_error.stderr b/lints/ui/format_error.stderr index 2c1feaa4884aa..adb37afdf6883 100644 --- a/lints/ui/format_error.stderr +++ b/lints/ui/format_error.stderr @@ -118,7 +118,7 @@ error: should not format error directly LL | info!("{}", err); | ^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:32:20 @@ -126,7 +126,7 @@ error: should not format error directly LL | my_info!("{}", err); | ^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:34:29 @@ -134,7 +134,7 @@ error: should not format error directly LL | tracing::field::display(&err); | ^^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:35:27 @@ -142,7 +142,7 @@ error: should not format error directly LL | tracing::field::debug(err.clone()); | ^^^^^^^^^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:37:5 @@ -150,7 +150,7 @@ error: should not format error directly LL | info!(%err, "233"); | ^^^^^^^^^^^^^^^^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:38:5 @@ -158,7 +158,7 @@ error: should not format error directly LL | info!(?err, "233"); | ^^^^^^^^^^^^^^^^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:39:23 @@ -166,7 +166,7 @@ error: should not format error directly LL | info!(%err, "{}", err); | ^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:39:5 @@ -174,7 +174,7 @@ error: should not format error directly LL | info!(%err, "{}", err); | ^^^^^^^^^^^^^^^^^^^^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:40:13 @@ -182,7 +182,7 @@ error: should not format error directly LL | let _ = info_span!("span", %err); | ^^^^^^^^^^^^^^^^^^^^^^^^ | - = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead error: should not format error directly --> $DIR/format_error.rs:44:9 @@ -240,5 +240,141 @@ LL | let _ = (&&err).to_string(); | = help: consider importing `thiserror_ext::AsReport` and using `.to_report_string()` instead -error: aborting due to 30 previous errors +error: should not format error directly + --> $DIR/format_error.rs:54:27 + | +LL | let _ = anyhow!("{}", err); + | ^^^ + | + = help: consider directly wrapping the error with `anyhow::anyhow!(..)` instead of formatting it + +error: should not format error directly + --> $DIR/format_error.rs:55:29 + | +LL | let _ = anyhow!("{:?}", err); + | ^^^ + | + = help: consider directly wrapping the error with `anyhow::anyhow!(..)` instead of formatting it + +error: should not format error directly + --> $DIR/format_error.rs:56:48 + | +LL | let _ = anyhow!("some error occurred: {}", err); + | ^^^ + | + = help: consider using `anyhow::Context::(with_)context` to attach additional message to the error and make it an error source instead + +error: should not format error directly + --> $DIR/format_error.rs:57:50 + | +LL | let _ = anyhow!("some error occurred: {:?}", err); + | ^^^ + | + = help: consider using `anyhow::Context::(with_)context` to attach additional message to the error and make it an error source instead + +error: should not format error directly + --> $DIR/format_error.rs:63:27 + | +LL | let _ = format!("{}", anyhow_err); + | ^^^^^^^^^^ + | + = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + +error: should not format error directly + --> $DIR/format_error.rs:64:27 + | +LL | let _ = format!("{}", &anyhow_err); + | ^^^^^^^^^^^ + | + = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + +error: should not format error directly + --> $DIR/format_error.rs:65:27 + | +LL | let _ = format!("{}", &&anyhow_err); + | ^^^^^^^^^^^^ + | + = help: consider importing `thiserror_ext::AsReport` and using `.as_report()` instead + +error: should not format error directly + --> $DIR/format_error.rs:68:29 + | +LL | tracing::field::display(&anyhow_err); + | ^^^^^^^^^^^ + | + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead + +error: should not format error directly + --> $DIR/format_error.rs:69:27 + | +LL | tracing::field::debug(make_anyhow_err()); + | ^^^^^^^^^^^^^^^^^ + | + = help: consider importing `thiserror_ext::AsReport` and recording the error as a field with `error = %.as_report()` instead + +error: should not format error directly + --> $DIR/format_error.rs:71:24 + | +LL | let _ = anyhow_err.to_string(); + | ^^^^^^^^^^^ + | + = help: consider importing `thiserror_ext::AsReport` and using `.to_report_string()` instead + +error: should not format error directly + --> $DIR/format_error.rs:72:28 + | +LL | let _ = (&&anyhow_err).to_string(); + | ^^^^^^^^^^^ + | + = help: consider importing `thiserror_ext::AsReport` and using `.to_report_string()` instead + +error: should not format error directly + --> $DIR/format_error.rs:74:27 + | +LL | let _ = anyhow!("{}", anyhow_err); + | ^^^^^^^^^^ + | + = help: consider removing the redundant wrapping of `anyhow::anyhow!(..)` + +error: should not format error directly + --> $DIR/format_error.rs:75:50 + | +LL | let _ = anyhow!("some error occurred: {:?}", anyhow_err); + | ^^^^^^^^^^ + | + = help: consider using `.context(..)` to attach additional message to the error and make it an error source instead + +error: should not format error directly + --> $DIR/format_error.rs:79:27 + | +LL | let _ = anyhow!("{}", err.as_report()); + | ^^^^^^^^^^^^^^^ + | + = help: consider directly wrapping the error with `anyhow::anyhow!(..)` instead of formatting its report + +error: should not format error directly + --> $DIR/format_error.rs:80:48 + | +LL | let _ = anyhow!("some error occurred: {}", err.as_report()); + | ^^^^^^^^^^^^^^^ + | + = help: consider using `anyhow::Context::(with_)context` to attach additional message to the error and make it an error source instead + +error: should not format error directly + --> $DIR/format_error.rs:81:29 + | +LL | let _ = anyhow!("{:?}", anyhow_err.as_report()); + | ^^^^^^^^^^^^^^^^^^^^^^ + | + = help: consider directly wrapping the error with `anyhow::anyhow!(..)` instead of formatting its report + +error: should not format error directly + --> $DIR/format_error.rs:82:50 + | +LL | let _ = anyhow!("some error occurred: {:?}", anyhow_err.as_report()); + | ^^^^^^^^^^^^^^^^^^^^^^ + | + = help: consider using `anyhow::Context::(with_)context` to attach additional message to the error and make it an error source instead + +error: aborting due to 47 previous errors diff --git a/proto/batch_plan.proto b/proto/batch_plan.proto index f6164f12226bf..b6c00b1a14aa5 100644 --- a/proto/batch_plan.proto +++ b/proto/batch_plan.proto @@ -59,7 +59,7 @@ message SourceNode { uint32 source_id = 1; repeated plan_common.ColumnCatalog columns = 2; map with_properties = 3; - bytes split = 4; + repeated bytes split = 4; catalog.StreamSourceInfo info = 5; } @@ -241,6 +241,8 @@ message ExchangeSource { message ExchangeNode { repeated ExchangeSource sources = 1; + // sequential means each tasks of the exchange node will be executed sequentially. + bool sequential = 2; repeated plan_common.Field input_schema = 3; } diff --git a/proto/buf.yaml b/proto/buf.yaml index 1aa31816ce0af..abad30f04506c 100644 --- a/proto/buf.yaml +++ b/proto/buf.yaml @@ -1,7 +1,8 @@ version: v1 breaking: use: - - WIRE # https://docs.buf.build/breaking/rules + - WIRE_JSON # https://docs.buf.build/breaking/rules + # https://github.com/risingwavelabs/risingwave/issues/15030 lint: use: - DEFAULT diff --git a/proto/catalog.proto b/proto/catalog.proto index ec7c68a3802ba..67e71848e3d00 100644 --- a/proto/catalog.proto +++ b/proto/catalog.proto @@ -159,6 +159,26 @@ message Sink { optional string created_at_cluster_version = 23; } +message Subscription { + uint32 id = 1; + string name = 2; + string definition = 3; + repeated common.ColumnOrder plan_pk = 4; + repeated int32 distribution_key = 5; + map properties = 6; + repeated plan_common.ColumnCatalog column_catalogs = 7; + uint32 database_id = 8; + uint32 schema_id = 9; + repeated uint32 dependent_relations = 10; + optional uint64 initialized_at_epoch = 11; + optional uint64 created_at_epoch = 12; + uint32 owner = 13; + StreamJobStatus stream_job_status = 14; + + optional string initialized_at_cluster_version = 15; + optional string created_at_cluster_version = 16; +} + message Connection { message PrivateLinkService { enum PrivateLinkProvider { @@ -213,12 +233,17 @@ message Function { uint32 database_id = 3; string name = 4; uint32 owner = 9; + repeated string arg_names = 15; repeated data.DataType arg_types = 5; data.DataType return_type = 6; string language = 7; - string link = 8; - string identifier = 10; + optional string link = 8; + optional string identifier = 10; + // The source code of the function. optional string body = 14; + // The zstd-compressed binary of the function. + optional bytes compressed_binary = 17; + bool always_retry_on_network_error = 16; oneof kind { ScalarFunction scalar = 11; @@ -265,7 +290,8 @@ message Table { repeated int32 stream_key = 13; bool append_only = 14; uint32 owner = 15; - map properties = 16; + reserved 16; + reserved "properties"; // deprecated uint32 fragment_id = 17; // an optional column index which is the vnode of each row computed by the // table's consistent hash distribution @@ -314,6 +340,9 @@ message Table { optional string initialized_at_cluster_version = 35; optional string created_at_cluster_version = 36; + // TTL of the record in the table, to ensure the consistency with other tables in the streaming plan, it only applies to append-only tables. + optional uint32 retention_seconds = 37; + // Per-table catalog version, used by schema change. `None` for internal tables and tests. // Not to be confused with the global catalog version for notification service. TableVersion version = 100; diff --git a/proto/connector_service.proto b/proto/connector_service.proto index 49fca31d1330d..a03af6305192b 100644 --- a/proto/connector_service.proto +++ b/proto/connector_service.proto @@ -156,12 +156,15 @@ message SinkCoordinatorStreamResponse { /* Source Service */ message CdcMessage { + // The value of the Debezium message string payload = 1; string partition = 2; string offset = 3; string full_table_name = 4; int64 source_ts_ms = 5; bool is_transaction_meta = 6; + // The key of the Debezium message, which only used by `mongodb-cdc` connector. + string key = 7; } enum SourceType { @@ -169,6 +172,7 @@ enum SourceType { MYSQL = 1; POSTGRES = 2; CITUS = 3; + MONGODB = 4; } message SourceCommonParam { diff --git a/proto/ddl_service.proto b/proto/ddl_service.proto index 1b584a7df78e1..78f4f3c818e41 100644 --- a/proto/ddl_service.proto +++ b/proto/ddl_service.proto @@ -98,6 +98,26 @@ message DropSinkResponse { uint64 version = 2; } +message CreateSubscriptionRequest { + catalog.Subscription subscription = 1; + stream_plan.StreamFragmentGraph fragment_graph = 2; +} + +message CreateSubscriptionResponse { + common.Status status = 1; + uint64 version = 2; +} + +message DropSubscriptionRequest { + uint32 subscription_id = 1; + bool cascade = 2; +} + +message DropSubscriptionResponse { + common.Status status = 1; + uint64 version = 2; +} + message CreateMaterializedViewRequest { catalog.Table materialized_view = 1; stream_plan.StreamFragmentGraph fragment_graph = 2; @@ -179,6 +199,7 @@ message AlterNameRequest { uint32 source_id = 5; uint32 schema_id = 6; uint32 database_id = 7; + uint32 subscription_id = 8; } string new_name = 20; } @@ -196,6 +217,7 @@ message AlterOwnerRequest { uint32 sink_id = 4; uint32 schema_id = 5; uint32 database_id = 6; + uint32 subscription_id = 7; } uint32 owner_id = 20; } @@ -208,6 +230,7 @@ message AlterSetSchemaRequest { uint32 sink_id = 4; uint32 function_id = 5; uint32 connection_id = 6; + uint32 subscription_id = 7; } uint32 new_schema_id = 20; } @@ -220,6 +243,7 @@ message AlterSetSchemaResponse { message AlterParallelismRequest { uint32 table_id = 1; meta.TableParallelism parallelism = 2; + bool deferred = 3; } message AlterParallelismResponse {} @@ -398,7 +422,9 @@ service DdlService { rpc CreateSource(CreateSourceRequest) returns (CreateSourceResponse); rpc DropSource(DropSourceRequest) returns (DropSourceResponse); rpc CreateSink(CreateSinkRequest) returns (CreateSinkResponse); + rpc CreateSubscription(CreateSubscriptionRequest) returns (CreateSubscriptionResponse); rpc DropSink(DropSinkRequest) returns (DropSinkResponse); + rpc DropSubscription(DropSubscriptionRequest) returns (DropSubscriptionResponse); rpc CreateMaterializedView(CreateMaterializedViewRequest) returns (CreateMaterializedViewResponse); rpc DropMaterializedView(DropMaterializedViewRequest) returns (DropMaterializedViewResponse); rpc CreateTable(CreateTableRequest) returns (CreateTableResponse); diff --git a/proto/expr.proto b/proto/expr.proto index 9c6dd8e59fbfc..9577018876590 100644 --- a/proto/expr.proto +++ b/proto/expr.proto @@ -49,18 +49,21 @@ message ExprNode { BITWISE_NOT = 34; BITWISE_SHIFT_LEFT = 35; BITWISE_SHIFT_RIGHT = 36; - // date functions + // date/time functions EXTRACT = 101; DATE_PART = 102; TUMBLE_START = 103; + MAKE_DATE = 113; + MAKE_TIME = 114; + MAKE_TIMESTAMP = 115; // From f64 to timestamp. // e.g. `select to_timestamp(1672044740.0)` - TO_TIMESTAMP = 104; + SEC_TO_TIMESTAMPTZ = 104; AT_TIME_ZONE = 105; DATE_TRUNC = 106; // Parse text to timestamp by format string. // e.g. `select to_timestamp('2022 08 21', 'YYYY MM DD')` - TO_TIMESTAMP1 = 107; + CHAR_TO_TIMESTAMPTZ = 107; CHAR_TO_DATE = 111; // Performs a cast with additional timezone information. CAST_WITH_TIME_ZONE = 108; @@ -82,6 +85,9 @@ message ExprNode { LTRIM = 210; RTRIM = 211; CASE = 212; + // Optimize case-when expression to constant lookup + // when arms are in a large scale with simple form + CONSTANT_LOOKUP = 624; // ROUND(numeric, integer) -> numeric ROUND_DIGIT = 213; // ROUND(numeric) -> numeric @@ -180,6 +186,8 @@ message ExprNode { PGWIRE_RECV = 321; CONVERT_FROM = 322; CONVERT_TO = 323; + DECRYPT = 324; + ENCRYPT = 325; // Unary operators NEG = 401; @@ -275,6 +283,10 @@ message ExprNode { PG_GET_INDEXDEF = 2400; COL_DESCRIPTION = 2401; PG_GET_VIEWDEF = 2402; + PG_GET_USERBYID = 2403; + PG_INDEXES_SIZE = 2404; + PG_RELATION_SIZE = 2405; + PG_GET_SERIAL_SEQUENCE = 2406; // EXTERNAL ICEBERG_TRANSFORM = 2201; @@ -410,9 +422,11 @@ message AggCall { message WindowFrame { enum Type { TYPE_UNSPECIFIED = 0; - // RANGE = 1; - TYPE_ROWS = 2; - // GROUPS = 3; + + TYPE_ROWS_LEGACY = 2 [deprecated = true]; // Deprecated since we introduced `RANGE` frame. + + TYPE_ROWS = 5; + TYPE_RANGE = 10; } enum BoundType { BOUND_TYPE_UNSPECIFIED = 0; @@ -422,7 +436,9 @@ message WindowFrame { BOUND_TYPE_FOLLOWING = 4; BOUND_TYPE_UNBOUNDED_FOLLOWING = 5; } + // Deprecated since we introduced `RANGE` frame. message Bound { + option deprecated = true; BoundType type = 1; oneof offset { uint64 integer = 2; @@ -436,11 +452,38 @@ message WindowFrame { // EXCLUSION_TIES = 3; EXCLUSION_NO_OTHERS = 4; } + message RowsFrameBounds { + RowsFrameBound start = 1; + RowsFrameBound end = 2; + } + message RowsFrameBound { + BoundType type = 1; + optional uint64 offset = 3; + } + message RangeFrameBounds { + RangeFrameBound start = 1; + RangeFrameBound end = 2; + + data.DataType order_data_type = 10; + common.OrderType order_type = 15; + data.DataType offset_data_type = 20; + } + message RangeFrameBound { + BoundType type = 1; + optional data.Datum offset = 3; + } Type type = 1; - Bound start = 2; - Bound end = 3; + + Bound start = 2 [deprecated = true]; // Deprecated since we introduced `RANGE` frame. + Bound end = 3 [deprecated = true]; // Deprecated since we introduced `RANGE` frame. + Exclusion exclusion = 4; + + oneof bounds { + RowsFrameBounds rows = 10; + RangeFrameBounds range = 15; + } } message WindowFunction { @@ -471,21 +514,30 @@ message WindowFunction { message UserDefinedFunction { repeated ExprNode children = 1; string name = 2; + repeated string arg_names = 8; repeated data.DataType arg_types = 3; string language = 4; - // For external UDF: the link to the external function service. - // For WASM UDF: the link to the wasm binary file. - string link = 5; - // An unique identifier for the function. - // For external UDF, it's the name of the function in the external function service. - // For WASM UDF, it's the name of the function in the wasm binary file. - string identifier = 6; + // The link to the external function service. + optional string link = 5; + // An unique identifier to the function. + // - If `link` is not empty, the name of the function in the external function service. + // - If `language` is `rust` or `wasm`, the name of the function in the wasm binary file. + // - If `language` is `javascript`, the name of the function. + optional string identifier = 6; + // - If `language` is `javascript`, the source code of the function. + optional string body = 7; + // - If `language` is `rust` or `wasm`, the zstd-compressed wasm binary. + optional bytes compressed_binary = 10; + bool always_retry_on_network_error = 9; } // Additional information for user defined table functions. message UserDefinedTableFunction { + repeated string arg_names = 8; repeated data.DataType arg_types = 3; string language = 4; - string link = 5; - string identifier = 6; + optional string link = 5; + optional string identifier = 6; + optional string body = 7; + optional bytes compressed_binary = 10; } diff --git a/proto/hummock.proto b/proto/hummock.proto index 1ff11077645d9..d40b31c52f60c 100644 --- a/proto/hummock.proto +++ b/proto/hummock.proto @@ -186,6 +186,12 @@ message HummockVersionCheckpoint { map stale_objects = 2; } +message HummockVersionArchive { + HummockVersion version = 1; + // some version_deltas since version + repeated HummockVersionDelta version_deltas = 2; +} + // We will have two epoch after decouple message HummockSnapshot { // Epoch with checkpoint, we will read durable data with it. @@ -278,7 +284,8 @@ message KeyRange { } message TableOption { - uint32 retention_seconds = 1; + reserved 1; + optional uint32 retention_seconds = 2; } message CompactTask { diff --git a/proto/meta.proto b/proto/meta.proto index 3e41032b33bfa..4ac73e3a0768d 100644 --- a/proto/meta.proto +++ b/proto/meta.proto @@ -373,6 +373,7 @@ message MetaSnapshot { repeated catalog.View views = 7; repeated catalog.Function functions = 15; repeated catalog.Connection connections = 17; + repeated catalog.Subscription subscriptions = 19; repeated user.UserInfo users = 8; // for streaming repeated FragmentParallelUnitMapping parallel_unit_mappings = 9; @@ -394,6 +395,7 @@ message Relation { catalog.Sink sink = 3; catalog.Index index = 4; catalog.View view = 5; + catalog.Subscription subscription = 6; } } @@ -469,14 +471,18 @@ message TableParallelism { uint32 parallelism = 1; } + // deprecated, treated as AdaptiveParallelism message AutoParallelism {} + message AdaptiveParallelism {} + message CustomParallelism {} oneof parallelism { FixedParallelism fixed = 1; AutoParallelism auto = 2; CustomParallelism custom = 3; + AdaptiveParallelism adaptive = 4; } } @@ -551,7 +557,7 @@ message SystemParams { optional uint32 parallel_compact_size_mb = 11; optional uint32 max_concurrent_creating_streaming_jobs = 12; optional bool pause_on_next_bootstrap = 13; - optional string wasm_storage_url = 14; + optional string wasm_storage_url = 14 [deprecated = true]; optional bool enable_tracing = 15; } diff --git a/proto/monitor_service.proto b/proto/monitor_service.proto index 7c7769da6b7ff..aa9880917d725 100644 --- a/proto/monitor_service.proto +++ b/proto/monitor_service.proto @@ -48,10 +48,25 @@ message AnalyzeHeapResponse { bytes result = 1; } +// Back pressure +message GetBackPressureRequest {} + +message BackPressureInfo { + uint32 actor_id = 1; + uint32 fragment_id = 2; + uint32 downstream_fragment_id = 3; + double value = 4; +} + +message GetBackPressureResponse { + repeated BackPressureInfo back_pressure_infos = 1; +} + service MonitorService { rpc StackTrace(StackTraceRequest) returns (StackTraceResponse); rpc Profiling(ProfilingRequest) returns (ProfilingResponse); rpc HeapProfiling(HeapProfilingRequest) returns (HeapProfilingResponse); rpc ListHeapProfiling(ListHeapProfilingRequest) returns (ListHeapProfilingResponse); rpc AnalyzeHeap(AnalyzeHeapRequest) returns (AnalyzeHeapResponse); + rpc GetBackPressure(GetBackPressureRequest) returns (GetBackPressureResponse); } diff --git a/proto/plan_common.proto b/proto/plan_common.proto index 6fdcc86feb7d8..79a1b1622704e 100644 --- a/proto/plan_common.proto +++ b/proto/plan_common.proto @@ -15,17 +15,6 @@ message Field { string name = 2; } -enum AdditionalColumnType { - UNSPECIFIED = 0; - KEY = 1; - TIMESTAMP = 2; - PARTITION = 3; - OFFSET = 4; - HEADER = 5; - FILENAME = 6; - NORMAL = 7; -} - enum ColumnDescVersion { COLUMN_DESC_VERSION_UNSPECIFIED = 0; // Introduced in https://github.com/risingwavelabs/risingwave/pull/13707#discussion_r1429947537, @@ -64,9 +53,13 @@ message ColumnDesc { // This field is used to represent the connector-spec additional column type. // UNSPECIFIED or unset for normal column. + + // deprecated, use AdditionalColumn instead, keep for compatibility with v1.6.x AdditionalColumnType additional_column_type = 9; ColumnDescVersion version = 10; + + AdditionalColumn additional_column = 11; } message ColumnCatalog { @@ -93,7 +86,7 @@ message StorageTableDesc { // TODO: may refactor primary key representations repeated common.ColumnOrder pk = 3; repeated uint32 dist_key_in_pk_indices = 4; - uint32 retention_seconds = 5; + reserved 5; repeated uint32 value_indices = 6; uint32 read_prefix_len_hint = 7; // Whether the table is versioned. If `true`, column-aware row encoding will @@ -101,6 +94,7 @@ message StorageTableDesc { bool versioned = 8; repeated uint32 stream_key = 9; optional uint32 vnode_col_idx_in_pk = 10; + optional uint32 retention_seconds = 11; } // Represents a table in external database for CDC scenario @@ -140,6 +134,7 @@ enum FormatType { FORMAT_TYPE_CANAL = 5; FORMAT_TYPE_UPSERT = 6; FORMAT_TYPE_PLAIN = 7; + FORMAT_TYPE_NONE = 8; } enum EncodeType { @@ -151,6 +146,7 @@ enum EncodeType { ENCODE_TYPE_JSON = 5; ENCODE_TYPE_BYTES = 6; ENCODE_TYPE_TEMPLATE = 7; + ENCODE_TYPE_NONE = 8; } enum RowFormatType { @@ -190,3 +186,44 @@ message Cardinality { message ExprContext { string time_zone = 1; } + +message AdditionalColumnKey {} + +message AdditionalColumnTimestamp {} + +message AdditionalColumnPartition {} + +message AdditionalColumnOffset {} + +message AdditionalColumnFilename {} + +message AdditionalColumnHeader { + string inner_field = 1; + data.DataType data_type = 2; +} + +// this type means we read all headers as a whole +message AdditionalColumnHeaders {} + +message AdditionalColumn { + oneof column_type { + AdditionalColumnKey key = 1; + AdditionalColumnTimestamp timestamp = 2; + AdditionalColumnPartition partition = 3; + AdditionalColumnOffset offset = 4; + AdditionalColumnHeader header_inner = 5; + AdditionalColumnFilename filename = 6; + AdditionalColumnHeaders headers = 7; + } +} + +enum AdditionalColumnType { + ADDITIONAL_COLUMN_TYPE_UNSPECIFIED = 0; + ADDITIONAL_COLUMN_TYPE_KEY = 1; + ADDITIONAL_COLUMN_TYPE_TIMESTAMP = 2; + ADDITIONAL_COLUMN_TYPE_PARTITION = 3; + ADDITIONAL_COLUMN_TYPE_OFFSET = 4; + ADDITIONAL_COLUMN_TYPE_HEADER = 5; + ADDITIONAL_COLUMN_TYPE_FILENAME = 6; + ADDITIONAL_COLUMN_TYPE_NORMAL = 7; +} diff --git a/proto/stream_plan.proto b/proto/stream_plan.proto index e69a712c9e3d8..359a0de15979b 100644 --- a/proto/stream_plan.proto +++ b/proto/stream_plan.proto @@ -223,6 +223,7 @@ message SinkDesc { string sink_from_name = 12; catalog.SinkFormatDesc format_desc = 13; optional uint32 target_table = 14; + optional uint64 extra_partition_col_idx = 15; } enum SinkLogStoreType { @@ -256,7 +257,6 @@ message FilterNode { message CdcFilterNode { expr.ExprNode search_condition = 1; uint32 upstream_source_id = 2; - repeated int32 upstream_column_ids = 3; } // A materialized view is regarded as a table. @@ -553,11 +553,17 @@ message StreamCdcScanNode { // Strips the primary key columns if they're unnecessary. repeated uint32 output_indices = 3; - /// The state table used by Backfill operator for persisting internal state + // The state table used by CdcBackfill operator for persisting internal state catalog.Table state_table = 4; // The external table that will be backfilled for CDC. plan_common.ExternalTableDesc cdc_table_desc = 5; + + // The rate limit for the stream cdc scan node. + optional uint32 rate_limit = 6; + + // Whether skip the backfill and only consume from upstream. + bool disable_backfill = 7; } // BatchPlanNode is used for mv on mv snapshot read. @@ -714,6 +720,12 @@ message OverWindowNode { OverWindowCachePolicy cache_policy = 5; } +message SubscriptionNode { + catalog.Subscription subscription_catalog = 1; + // log store should have a table. + catalog.Table log_store_table = 2; +} + message StreamNode { oneof node_body { SourceNode source = 100; @@ -757,6 +769,7 @@ message StreamNode { StreamFsFetchNode stream_fs_fetch = 138; StreamCdcScanNode stream_cdc_scan = 139; CdcFilterNode cdc_filter = 140; + SubscriptionNode subscription = 141; } // The id for the operator. This is local per mview. // TODO: should better be a uint32. @@ -851,6 +864,7 @@ enum FragmentTypeFlag { FRAGMENT_TYPE_FLAG_VALUES = 64; FRAGMENT_TYPE_FLAG_DML = 128; FRAGMENT_TYPE_FLAG_CDC_FILTER = 256; + FRAGMENT_TYPE_FLAG_SUBSCRIPTION = 512; } // The streaming context associated with a stream plan diff --git a/proto/stream_service.proto b/proto/stream_service.proto index 462f5ff0256a6..e8c5d94a20ac3 100644 --- a/proto/stream_service.proto +++ b/proto/stream_service.proto @@ -46,6 +46,7 @@ message DropActorsResponse { message ForceStopActorsRequest { string request_id = 1; + uint64 prev_epoch = 2; } message ForceStopActorsResponse { diff --git a/proto/telemetry.proto b/proto/telemetry.proto new file mode 100644 index 0000000000000..3fcf3911ea361 --- /dev/null +++ b/proto/telemetry.proto @@ -0,0 +1,97 @@ +syntax = "proto3"; + +package telemetry; + +option go_package = "risingwavelabs.com/risingwave/proto/telemetry"; + +enum MetaBackend { + META_BACKEND_UNSPECIFIED = 0; + META_BACKEND_MEMORY = 1; + META_BACKEND_ETCD = 2; + META_BACKEND_RDB = 3; +} + +enum TelemetryNodeType { + TELEMETRY_NODE_TYPE_UNSPECIFIED = 0; + TELEMETRY_NODE_TYPE_META = 1; + TELEMETRY_NODE_TYPE_COMPUTE = 2; + TELEMETRY_NODE_TYPE_FRONTEND = 3; + TELEMETRY_NODE_TYPE_COMPACTOR = 4; +} + +message SystemMemory { + uint64 used = 1; + uint64 total = 2; +} + +message SystemOs { + string name = 1; + string version = 2; + string kernel_version = 3; +} + +message SystemCpu { + float available = 1; +} + +message SystemData { + SystemMemory memory = 1; + SystemOs os = 2; + SystemCpu cpu = 3; +} + +// NodeCount represents how many nodes in this cluster +message NodeCount { + uint32 meta = 1; + uint32 compute = 2; + uint32 frontend = 3; + uint32 compactor = 4; +} + +// RwVersion represents the version of RisingWave +message RwVersion { + // Version is the Cargo package version of RisingWave + string rw_version = 1; + // GitSHA is the Git commit SHA of RisingWave + string git_sha = 2; +} + +message ReportBase { + // tracking_id is persistent in meta data + string tracking_id = 1; + // session_id is reset every time node restarts + string session_id = 2; + // system_data is hardware and os info + SystemData system_data = 3; + // up_time is how long the node has been running + uint64 up_time = 4; + // report_time is when the report is created + uint64 report_time = 5; + // node_type is the node that creates the report + TelemetryNodeType node_type = 6; +} + +message MetaReport { + ReportBase base = 1; + // meta_backend is the backend of meta data + MetaBackend meta_backend = 2; + // node_count is the count of each node type + NodeCount node_count = 3; + // rw_version is the version of RisingWave + RwVersion rw_version = 4; + // This field represents the "number of running streaming jobs" + // and is used to indicate whether the cluster is active. + uint32 stream_job_count = 5; +} + +message ComputeReport { + ReportBase base = 1; +} + +message FrontendReport { + ReportBase base = 1; +} + +message CompactorReport { + ReportBase base = 1; +} diff --git a/proto/user.proto b/proto/user.proto index 69661d46f0db3..383e78cb57b28 100644 --- a/proto/user.proto +++ b/proto/user.proto @@ -14,9 +14,11 @@ message AuthInfo { PLAINTEXT = 1; SHA256 = 2; MD5 = 3; + OAUTH = 4; } EncryptionType encryption_type = 1; bytes encrypted_value = 2; + map metadata = 3; } // User defines a user in the system. @@ -65,6 +67,7 @@ message GrantPrivilege { uint32 all_tables_schema_id = 11; uint32 all_sources_schema_id = 12; uint32 all_dml_tables_schema_id = 13; + uint32 subscription_id = 14; } repeated ActionWithGrantOption action_with_opts = 7; } diff --git a/risedev.yml b/risedev.yml index 17076e4f2da8e..cb352daab6cf9 100644 --- a/risedev.yml +++ b/risedev.yml @@ -164,6 +164,17 @@ profile: - use: compactor # - use: prometheus # - use: grafana + fs: + steps: + # - use: etcd + - use: meta-node + - use: compute-node + - use: frontend + - use: opendal + engine: fs + - use: compactor + # - use: prometheus + # - use: grafana webhdfs: steps: # - use: etcd @@ -541,6 +552,80 @@ profile: - use: frontend - use: compactor + ci-3cn-1fe-with-monitoring: + config-path: src/config/ci.toml + steps: + - use: minio + - use: etcd + unsafe-no-fsync: true + - use: meta-node + - use: compute-node + port: 5687 + exporter-port: 1222 + enable-tiered-cache: true + - use: compute-node + port: 5688 + exporter-port: 1223 + enable-tiered-cache: true + - use: compute-node + port: 5689 + exporter-port: 1224 + enable-tiered-cache: true + - use: frontend + - use: compactor + - use: prometheus + - use: grafana + + ci-3cn-1fe-with-minio-rate-limit: + config-path: src/config/ci.toml + steps: + - use: minio + api-requests-max: 30 + api-requests-deadline: 3s + - use: etcd + unsafe-no-fsync: true + - use: meta-node + - use: compute-node + port: 5687 + exporter-port: 1222 + enable-tiered-cache: true + - use: compute-node + port: 5688 + exporter-port: 1223 + enable-tiered-cache: true + - use: compute-node + port: 5689 + exporter-port: 1224 + enable-tiered-cache: true + - use: frontend + - use: compactor + + ci-3cn-1fe-with-monitoring-and-minio-rate-limit: + config-path: src/config/ci.toml + steps: + - use: minio + api-requests-max: 30 + api-requests-deadline: 2s + - use: etcd + unsafe-no-fsync: true + - use: meta-node + - use: compute-node + port: 5687 + exporter-port: 1222 + enable-tiered-cache: true + - use: compute-node + port: 5688 + exporter-port: 1223 + enable-tiered-cache: true + - use: compute-node + port: 5689 + exporter-port: 1224 + enable-tiered-cache: true + - use: frontend + - use: compactor + - use: prometheus + - use: grafana + ci-3cn-3fe: config-path: src/config/ci.toml steps: @@ -735,6 +820,28 @@ profile: - use: frontend - use: compactor + ci-3cn-1fe-with-recovery: + config-path: src/config/ci-recovery.toml + steps: + - use: minio + - use: etcd + unsafe-no-fsync: true + - use: meta-node + - use: compute-node + port: 5687 + exporter-port: 1222 + enable-tiered-cache: true + - use: compute-node + port: 5688 + exporter-port: 1223 + enable-tiered-cache: true + - use: compute-node + port: 5689 + exporter-port: 1224 + enable-tiered-cache: true + - use: frontend + - use: compactor + ci-1cn-1fe-kafka-with-recovery: config-path: src/config/ci-recovery.toml steps: @@ -766,11 +873,6 @@ profile: - use: etcd - use: minio - ci-delete-range-test: - config-path: src/config/ci-delete-range-test.toml - steps: - - use: minio - ci-iceberg-test: config-path: src/config/ci-iceberg-test.toml steps: @@ -781,27 +883,7 @@ profile: - use: frontend - use: compactor - ci-deltalake-test: - config-path: src/config/ci.toml - steps: - - use: minio - - use: meta-node - - use: compute-node - enable-tiered-cache: true - - use: frontend - - use: compactor - - ci-clickhouse-test: - config-path: src/config/ci.toml - steps: - - use: minio - - use: meta-node - - use: compute-node - enable-tiered-cache: true - - use: frontend - - use: compactor - - ci-pulsar-test: + ci-sink-test: config-path: src/config/ci.toml steps: - use: minio @@ -885,6 +967,15 @@ template: # Prometheus nodes used by this MinIO provide-prometheus: "prometheus*" + # Max concurrent api requests. + # see: https://github.com/minio/minio/blob/master/docs/throttle/README.md. + # '0' means this env var will use the default of minio. + api-requests-max: 0 + + # Deadline for api requests. + # Empty string means this env var will use the default of minio. + api-requests-deadline: "" + etcd: # Id of this instance id: etcd-${port} diff --git a/scripts/install/install-risingwave.sh b/scripts/install/install-risingwave.sh new file mode 100755 index 0000000000000..6d1440e9b6b2c --- /dev/null +++ b/scripts/install/install-risingwave.sh @@ -0,0 +1,105 @@ +#!/bin/sh -e + +if [ -z "${OS}" ]; then + OS=$(uname -s) +fi +if [ -z "${ARCH}" ]; then + ARCH=$(uname -m) +fi +STATE_STORE_PATH="${HOME}/.risingwave/state_store" +META_STORE_PATH="${HOME}/.risingwave/meta_store" + +VERSION="v1.7.0-standalone" +# TODO(kwannoel): re-enable it once we have stable release in latest for single node mode. +#VERSION=$(curl -s https://api.github.com/repos/risingwavelabs/risingwave/releases/latest \ +# | grep '.tag_name' \ +# | sed -E -n 's/.*(v[0-9]+.[0-9]+.[0-9])\",/\1/p') +HOMEBREW_VERSION="1.7-standalone" + +BASE_URL="https://github.com/risingwavelabs/risingwave/releases/download" + +if [ "${OS}" = "Linux" ]; then + if [ "${ARCH}" = "x86_64" ] || [ "${ARCH}" = "amd64" ]; then + BASE_ARCHIVE_NAME="risingwave-${VERSION}-x86_64-unknown-linux-all-in-one" + ARCHIVE_NAME="${BASE_ARCHIVE_NAME}.tar.gz" + URL="${BASE_URL}/${VERSION}/${ARCHIVE_NAME}" + USE_BREW=0 + elif [ "${ARCH}" = "arm64" ] || [ "${ARCH}" = "aarch64" ]; then + BASE_ARCHIVE_NAME="risingwave-${VERSION}-aarch64-unknown-linux-all-in-one" + ARCHIVE_NAME="${BASE_ARCHIVE_NAME}.tar.gz" + URL="${BASE_URL}/${VERSION}/${ARCHIVE_NAME}" + USE_BREW=0 + fi +elif [ "${OS}" = "Darwin" ]; then + if [ "${ARCH}" = "x86_64" ] || [ "${ARCH}" = "amd64" ] || [ "${ARCH}" = "aarch64" ] || [ "${ARCH}" = "arm64" ]; then + USE_BREW=1 + fi +fi + +if [ -z "$USE_BREW" ]; then + echo + echo "Unsupported OS or Architecture: ${OS}-${ARCH}" + echo + echo "Supported OSes: Linux, macOS" + echo "Supported architectures: x86_64" + echo + echo "Please open an issue at ," + echo "if you would like to request support for your OS or architecture." + echo + exit 1 +fi + +############# BREW INSTALL +if [ "${USE_BREW}" -eq 1 ]; then + echo "Installing RisingWave@${HOMEBREW_VERSION} using Homebrew." + brew tap risingwavelabs/risingwave + brew install risingwave@${HOMEBREW_VERSION} + echo + echo "Successfully installed RisingWave@${HOMEBREW_VERSION} using Homebrew." + echo + echo "Run RisingWave:" + echo + echo " risingwave" + echo + echo "Start a psql session:" + echo + echo " psql -h localhost -p 4566 -d dev -U root" + echo + echo "To start a fresh cluster, you can just delete the data directory contents:" + echo + echo " rm -r ~/.risingwave" + echo + echo "To view available options, run:" + echo + echo " risingwave single-node --help" + echo + echo + exit 0 +fi + +############# BINARY INSTALL +echo +echo "Downloading RisingWave@${VERSION} from ${URL} into ${PWD}." +echo +curl -L "${URL}" | tar -zx || exit 1 +chmod +x risingwave +echo +echo "Successfully installed RisingWave@${VERSION} binary." +echo +echo "Run RisingWave:" +echo +echo " ./risingwave" +echo +echo "Start a psql session:" +echo +echo " psql -h localhost -p 4566 -d dev -U root" +echo +echo "To start a fresh cluster, you can just delete the data directory contents:" +echo +echo " rm -r ~/.risingwave" +echo +echo "To view available options, run:" +echo +echo " risingwave single-node --help" +echo +# TODO(kwannoel): Include link to our docs. \ No newline at end of file diff --git a/scripts/source/prepare_ci_pubsub/Cargo.toml b/scripts/source/prepare_ci_pubsub/Cargo.toml index 7c5f8ce89bfc8..52d6e2460a3db 100644 --- a/scripts/source/prepare_ci_pubsub/Cargo.toml +++ b/scripts/source/prepare_ci_pubsub/Cargo.toml @@ -13,7 +13,7 @@ normal = ["workspace-hack"] [dependencies] anyhow = "1" google-cloud-googleapis = { version = "0.12", features = ["pubsub"] } -google-cloud-pubsub = "0.22" +google-cloud-pubsub = "0.23" tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", "rt-multi-thread", diff --git a/scripts/source/test_data/debezium_mess_key.1 b/scripts/source/test_data/debezium_mess_key.1 new file mode 100644 index 0000000000000..59b0b0e70defd --- /dev/null +++ b/scripts/source/test_data/debezium_mess_key.1 @@ -0,0 +1,4 @@ +****^{"before":null,"after":{"order_id":1,"order_date":1558430840000,"customer_name":"Bob","price":null,"product_id":1,"order_status":1},"source":{"version":"1.9.7.Final","connector":"postgresql","name":"postgres","ts_ms":1685975423896,"snapshot":"true","db":"mydb","sequence":"[null,\"23911624\"]","schema":"public","table":"orders","txId":490,"lsn":23911624,"xmin":null},"op":"r","ts_ms":1685975424096,"transaction":null} +****^{"before":null,"after":{"order_id":2,"order_date":1558430840001,"customer_name":"Alice","price":null,"product_id":2,"order_status":1},"source":{"version":"1.9.7.Final","connector":"postgresql","name":"postgres","ts_ms":1685975423896,"snapshot":"true","db":"mydb","sequence":"[null,\"23911624\"]","schema":"public","table":"orders","txId":490,"lsn":23911624,"xmin":null},"op":"r","ts_ms":1685975424102,"transaction":null} +****^{"before":null,"after":{"order_id":3,"order_date":1558430840002,"customer_name":"Alice","price":null,"product_id":2,"order_status":1},"source":{"version":"1.9.7.Final","connector":"postgresql","name":"postgres","ts_ms":1685975423896,"snapshot":"last","db":"mydb","sequence":"[null,\"23911624\"]","schema":"public","table":"orders","txId":490,"lsn":23911624,"xmin":null},"op":"r","ts_ms":1685975424103,"transaction":null} +****^{"before":null,"after":{"order_id":1,"order_date":1558430840000,"customer_name":"Bob","price":null,"product_id":3,"order_status":1},"source":{"version":"1.9.7.Final","connector":"postgresql","name":"postgres","ts_ms":1686029203809,"snapshot":"false","db":"mydb","sequence":"[null,\"23911952\"]","schema":"public","table":"orders","txId":491,"lsn":23911952,"xmin":null},"op":"u","ts_ms":1686029204058,"transaction":null} diff --git a/src/batch/Cargo.toml b/src/batch/Cargo.toml index 0d8d4b36a95ff..fa18fa0c83717 100644 --- a/src/batch/Cargo.toml +++ b/src/batch/Cargo.toml @@ -15,6 +15,8 @@ normal = ["workspace-hack"] [dependencies] anyhow = "1" +arrow-array = { workspace = true } +arrow-schema = { workspace = true } assert_matches = "1" async-recursion = "1" async-trait = "0.1" @@ -24,6 +26,7 @@ futures-async-stream = { workspace = true } futures-util = "0.3" hashbrown = { workspace = true } hytra = "0.1.2" +icelake = { workspace = true } itertools = "0.12" memcomparable = "0.2" parking_lot = { version = "0.12", features = ["arc_lock"] } @@ -31,12 +34,13 @@ paste = "1" prometheus = { version = "0.13", features = ["process"] } risingwave_common = { workspace = true } risingwave_connector = { workspace = true } +risingwave_dml = { workspace = true } risingwave_expr = { workspace = true } risingwave_hummock_sdk = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } -risingwave_source = { workspace = true } risingwave_storage = { workspace = true } +rw_futures_util = { workspace = true } scopeguard = "1" serde_json = "1" thiserror = "1" diff --git a/src/batch/benches/utils/mod.rs b/src/batch/benches/utils/mod.rs index 4abd48ffc0ffa..979675f2db135 100644 --- a/src/batch/benches/utils/mod.rs +++ b/src/batch/benches/utils/mod.rs @@ -76,11 +76,7 @@ pub fn create_input( chunk_num: usize, ) -> BoxedExecutor { let mut input = MockExecutor::new(Schema { - fields: input_types - .iter() - .map(Clone::clone) - .map(Field::unnamed) - .collect(), + fields: input_types.iter().cloned().map(Field::unnamed).collect(), }); for c in gen_data(chunk_size, chunk_num, input_types) { input.add(c); diff --git a/src/batch/clippy.toml b/src/batch/clippy.toml index aeb91fb713fb1..df03c275d0dee 100644 --- a/src/batch/clippy.toml +++ b/src/batch/clippy.toml @@ -1,11 +1,6 @@ -disallowed-methods = [ -] +disallowed-methods = [] -disallowed-types = [ - { path = "risingwave_common::error::ErrorCode", reason = "Please use per-crate error type instead." }, - { path = "risingwave_common::error::RwError", reason = "Please use per-crate error type instead." }, - { path = "risingwave_common::error::Result", reason = "Please use per-crate error type instead." }, -] +disallowed-types = [] doc-valid-idents = [ "RisingWave", @@ -16,7 +11,7 @@ doc-valid-idents = [ "PostgreSQL", "MySQL", "TopN", - "VNode" + "VNode", ] avoid-breaking-exported-api = false upper-case-acronyms-aggressive = true diff --git a/src/batch/src/error.rs b/src/batch/src/error.rs index d5efbbef9dd94..f4d7341cbc6dd 100644 --- a/src/batch/src/error.rs +++ b/src/batch/src/error.rs @@ -18,8 +18,10 @@ use std::sync::Arc; pub use anyhow::anyhow; use risingwave_common::array::ArrayError; -use risingwave_common::error::{BoxedError, ErrorCode, RwError}; +use risingwave_common::error::BoxedError; use risingwave_common::util::value_encoding::error::ValueEncodingError; +use risingwave_connector::error::ConnectorError; +use risingwave_dml::error::DmlError; use risingwave_expr::ExprError; use risingwave_pb::PbFieldNotFound; use risingwave_rpc_client::error::{RpcError, ToTonicStatus}; @@ -101,7 +103,22 @@ pub enum BatchError { BoxedError, ), + #[error(transparent)] + Dml( + #[from] + #[backtrace] + DmlError, + ), + + #[error(transparent)] + Iceberg( + #[from] + #[backtrace] + icelake::Error, + ), + // Make the ref-counted type to be a variant for easier code structuring. + // TODO(error-handling): replace with `thiserror_ext::Arc` #[error(transparent)] Shared( #[from] @@ -129,19 +146,6 @@ impl From for BatchError { } } -impl From for RwError { - fn from(s: BatchError) -> Self { - ErrorCode::BatchError(Box::new(s)).into() - } -} - -// TODO(error-handling): remove after eliminating RwError from connector. -impl From for BatchError { - fn from(s: RwError) -> Self { - Self::Internal(anyhow!(s)) - } -} - impl<'a> From<&'a BatchError> for Status { fn from(err: &'a BatchError) -> Self { err.to_status(tonic::Code::Internal, "batch") @@ -153,3 +157,9 @@ impl From for Status { Self::from(&err) } } + +impl From for BatchError { + fn from(value: ConnectorError) -> Self { + Self::Connector(value.into()) + } +} diff --git a/src/batch/src/executor/delete.rs b/src/batch/src/executor/delete.rs index c5d7d06c42335..00b7ffb1ca519 100644 --- a/src/batch/src/executor/delete.rs +++ b/src/batch/src/executor/delete.rs @@ -21,8 +21,8 @@ use risingwave_common::catalog::{Field, Schema, TableId, TableVersionId}; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::types::DataType; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; +use risingwave_dml::dml_manager::DmlManagerRef; use risingwave_pb::batch_plan::plan_node::NodeBody; -use risingwave_source::dml_manager::DmlManagerRef; use crate::error::{BatchError, Result}; use crate::executor::{ @@ -201,7 +201,7 @@ mod tests { schema_test_utils, ColumnDesc, ColumnId, INITIAL_TABLE_VERSION_ID, }; use risingwave_common::test_prelude::DataChunkTestExt; - use risingwave_source::dml_manager::DmlManager; + use risingwave_dml::dml_manager::DmlManager; use super::*; use crate::executor::test_utils::MockExecutor; diff --git a/src/batch/src/executor/generic_exchange.rs b/src/batch/src/executor/generic_exchange.rs index e54cb9069a393..4a50be2c2665a 100644 --- a/src/batch/src/executor/generic_exchange.rs +++ b/src/batch/src/executor/generic_exchange.rs @@ -14,15 +14,14 @@ use futures::StreamExt; use futures_async_stream::try_stream; -use itertools::Itertools; use risingwave_common::array::DataChunk; use risingwave_common::catalog::{Field, Schema}; use risingwave_common::util::iter_util::ZipEqFast; -use risingwave_common::util::select_all; use risingwave_pb::batch_plan::plan_node::NodeBody; use risingwave_pb::batch_plan::PbExchangeSource; use risingwave_pb::plan_common::Field as NodeField; use risingwave_rpc_client::ComputeClientPoolRef; +use rw_futures_util::select_all; use crate::error::{BatchError, Result}; use crate::exchange_source::ExchangeSourceImpl; @@ -39,6 +38,7 @@ pub struct GenericExchangeExecutor { proto_sources: Vec, /// Mock-able CreateSource. source_creators: Vec, + sequential: bool, context: C, schema: Schema, @@ -126,6 +126,8 @@ impl BoxedExecutorBuilder for GenericExchangeExecutorBuilder { NodeBody::Exchange )?; + let sequential = node.get_sequential(); + ensure!(!node.get_sources().is_empty()); let proto_sources: Vec = node.get_sources().to_vec(); let source_creators = @@ -136,6 +138,7 @@ impl BoxedExecutorBuilder for GenericExchangeExecutorBuilder { Ok(Box::new(ExchangeExecutor:: { proto_sources, source_creators, + sequential, context: source.context().clone(), schema: Schema { fields }, task_id: source.task_id.clone(), @@ -164,26 +167,33 @@ impl Executor impl GenericExchangeExecutor { #[try_stream(boxed, ok = DataChunk, error = BatchError)] async fn do_execute(self: Box) { - let mut stream = select_all( - self.proto_sources - .into_iter() - .zip_eq_fast(self.source_creators) - .map(|(prost_source, source_creator)| { - Self::data_chunk_stream( - prost_source, - source_creator, - self.context.clone(), - self.metrics.clone(), - self.identity.clone(), - ) - }) - .collect_vec(), - ) - .boxed(); - - while let Some(data_chunk) = stream.next().await { - let data_chunk = data_chunk?; - yield data_chunk + let streams = self + .proto_sources + .into_iter() + .zip_eq_fast(self.source_creators) + .map(|(prost_source, source_creator)| { + Self::data_chunk_stream( + prost_source, + source_creator, + self.context.clone(), + self.metrics.clone(), + self.identity.clone(), + ) + }); + + if self.sequential { + for mut stream in streams { + while let Some(data_chunk) = stream.next().await { + let data_chunk = data_chunk?; + yield data_chunk + } + } + } else { + let mut stream = select_all(streams).boxed(); + while let Some(data_chunk) = stream.next().await { + let data_chunk = data_chunk?; + yield data_chunk + } } } @@ -262,6 +272,7 @@ mod tests { metrics: None, proto_sources, source_creators, + sequential: false, context, schema: Schema { fields: vec![Field::unnamed(DataType::Int32)], diff --git a/src/batch/src/executor/group_top_n.rs b/src/batch/src/executor/group_top_n.rs index 997b593d24920..2b33ac6f28319 100644 --- a/src/batch/src/executor/group_top_n.rs +++ b/src/batch/src/executor/group_top_n.rs @@ -197,7 +197,7 @@ impl GroupTopNExecutor { #[for_await] for chunk in self.child.execute() { let chunk = Arc::new(chunk?); - let keys = K::build(self.group_key.as_slice(), &chunk)?; + let keys = K::build_many(self.group_key.as_slice(), &chunk); for (row_id, ((encoded_row, key), visible)) in encode_chunk(&chunk, &self.column_orders)? diff --git a/src/batch/src/executor/hash_agg.rs b/src/batch/src/executor/hash_agg.rs index a5df8d443e949..e4e6a32e618d8 100644 --- a/src/batch/src/executor/hash_agg.rs +++ b/src/batch/src/executor/hash_agg.rs @@ -224,7 +224,7 @@ impl HashAggExecutor { #[for_await] for chunk in self.child.execute() { let chunk = StreamChunk::from(chunk?); - let keys = K::build(self.group_key_columns.as_slice(), &chunk)?; + let keys = K::build_many(self.group_key_columns.as_slice(), &chunk); let mut memory_usage_diff = 0; for (row_id, (key, visible)) in keys .into_iter() diff --git a/src/batch/src/executor/iceberg_scan.rs b/src/batch/src/executor/iceberg_scan.rs new file mode 100644 index 0000000000000..9c24e554c8e1f --- /dev/null +++ b/src/batch/src/executor/iceberg_scan.rs @@ -0,0 +1,190 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::hash::{DefaultHasher, Hash, Hasher}; +use std::sync::Arc; + +use anyhow::anyhow; +use arrow_array::RecordBatch; +use futures_async_stream::try_stream; +use futures_util::stream::StreamExt; +use icelake::io::{FileScan, TableScan}; +use risingwave_common::catalog::Schema; +use risingwave_connector::sink::iceberg::IcebergConfig; + +use crate::error::BatchError; +use crate::executor::{DataChunk, Executor}; + +/// Create a iceberg scan executor. +/// +/// # Examples +/// +/// ``` +/// use futures_async_stream::for_await; +/// use risingwave_batch::executor::{Executor, FileSelector, IcebergScanExecutor}; +/// use risingwave_common::catalog::{Field, Schema}; +/// use risingwave_common::types::DataType; +/// use risingwave_connector::sink::iceberg::IcebergConfig; +/// +/// #[tokio::test] +/// async fn test_iceberg_scan() { +/// let iceberg_scan_executor = IcebergScanExecutor::new( +/// IcebergConfig { +/// database_name: "demo_db".into(), +/// table_name: "demo_table".into(), +/// catalog_type: Some("storage".into()), +/// path: "s3a://hummock001/".into(), +/// endpoint: Some("http://127.0.0.1:9301".into()), +/// access_key: "hummockadmin".into(), +/// secret_key: "hummockadmin".into(), +/// region: Some("us-east-1".into()), +/// ..Default::default() +/// }, +/// None, +/// FileSelector::select_all(), +/// 1024, +/// Schema::new(vec![ +/// Field::with_name(DataType::Int64, "seq_id"), +/// Field::with_name(DataType::Int64, "user_id"), +/// Field::with_name(DataType::Varchar, "user_name"), +/// ]), +/// "iceberg_scan".into(), +/// ); +/// +/// let stream = Box::new(iceberg_scan_executor).execute(); +/// #[for_await] +/// for chunk in stream { +/// let chunk = chunk.unwrap(); +/// println!("{:?}", chunk); +/// } +/// } +/// ``` + +pub struct IcebergScanExecutor { + iceberg_config: IcebergConfig, + snapshot_id: Option, + file_selector: FileSelector, + batch_size: usize, + + schema: Schema, + identity: String, +} + +impl Executor for IcebergScanExecutor { + fn schema(&self) -> &risingwave_common::catalog::Schema { + &self.schema + } + + fn identity(&self) -> &str { + &self.identity + } + + fn execute(self: Box) -> super::BoxedDataChunkStream { + self.do_execute().boxed() + } +} + +impl IcebergScanExecutor { + pub fn new( + iceberg_config: IcebergConfig, + snapshot_id: Option, + file_selector: FileSelector, + batch_size: usize, + schema: Schema, + identity: String, + ) -> Self { + Self { + iceberg_config, + snapshot_id, + file_selector, + batch_size, + schema, + identity, + } + } + + #[try_stream(ok = DataChunk, error = BatchError)] + async fn do_execute(self: Box) { + let table = self.iceberg_config.load_table().await?; + + let table_scan: TableScan = table + .new_scan_builder() + .with_snapshot_id( + self.snapshot_id + .unwrap_or_else(|| table.current_table_metadata().current_snapshot_id.unwrap()), + ) + .with_batch_size(self.batch_size) + .with_column_names(self.schema.names()) + .build() + .map_err(|e| BatchError::Internal(anyhow!(e)))?; + let file_scan_stream: icelake::io::FileScanStream = + table_scan.scan(&table).await.map_err(BatchError::Iceberg)?; + + #[for_await] + for file_scan in file_scan_stream { + let file_scan: FileScan = file_scan.map_err(BatchError::Iceberg)?; + if !self.file_selector.select(file_scan.path()) { + continue; + } + let record_batch_stream = file_scan.scan().await.map_err(BatchError::Iceberg)?; + + #[for_await] + for record_batch in record_batch_stream { + let record_batch: RecordBatch = record_batch.map_err(BatchError::Iceberg)?; + let chunk = Self::record_batch_to_chunk(record_batch)?; + debug_assert_eq!(chunk.data_types(), self.schema.data_types()); + yield chunk; + } + } + } + + fn record_batch_to_chunk(record_batch: RecordBatch) -> Result { + let mut columns = Vec::with_capacity(record_batch.num_columns()); + for array in record_batch.columns() { + let column = Arc::new(array.try_into()?); + columns.push(column); + } + Ok(DataChunk::new(columns, record_batch.num_rows())) + } +} + +pub enum FileSelector { + // File paths to be scanned by this executor are specified. + FileList(Vec), + // Data files to be scanned by this executor could be calculated by Hash(file_path) % num_tasks == task_id. + // task_id, num_tasks + Hash(usize, usize), +} + +impl FileSelector { + pub fn select_all() -> Self { + FileSelector::Hash(0, 1) + } + + pub fn select(&self, path: &str) -> bool { + match self { + FileSelector::FileList(paths) => paths.contains(&path.to_string()), + FileSelector::Hash(task_id, num_tasks) => { + let hash = Self::hash_str_to_usize(path); + hash % num_tasks == *task_id + } + } + } + + pub fn hash_str_to_usize(s: &str) -> usize { + let mut hasher = DefaultHasher::new(); + s.hash(&mut hasher); + hasher.finish() as usize + } +} diff --git a/src/batch/src/executor/insert.rs b/src/batch/src/executor/insert.rs index d236a25561029..951aec3573e9e 100644 --- a/src/batch/src/executor/insert.rs +++ b/src/batch/src/executor/insert.rs @@ -25,10 +25,10 @@ use risingwave_common::catalog::{Field, Schema, TableId, TableVersionId}; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::types::DataType; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; +use risingwave_dml::dml_manager::DmlManagerRef; use risingwave_expr::expr::{build_from_prost, BoxedExpression}; use risingwave_pb::batch_plan::plan_node::NodeBody; use risingwave_pb::plan_common::IndexAndExpr; -use risingwave_source::dml_manager::DmlManagerRef; use crate::error::{BatchError, Result}; use crate::executor::{ @@ -276,7 +276,7 @@ mod tests { }; use risingwave_common::transaction::transaction_message::TxnMsg; use risingwave_common::types::{DataType, StructType}; - use risingwave_source::dml_manager::DmlManager; + use risingwave_dml::dml_manager::DmlManager; use risingwave_storage::hummock::CachePolicy; use risingwave_storage::memory::MemoryStateStore; use risingwave_storage::store::{ReadOptions, StateStoreReadExt}; diff --git a/src/batch/src/executor/join/hash_join.rs b/src/batch/src/executor/join/hash_join.rs index ab1f6019f9878..bbebc114ae628 100644 --- a/src/batch/src/executor/join/hash_join.rs +++ b/src/batch/src/executor/join/hash_join.rs @@ -250,7 +250,7 @@ impl HashJoinExecutor { // Build hash map for (build_chunk_id, build_chunk) in build_side.iter().enumerate() { - let build_keys = K::build(&self.build_key_idxs, build_chunk)?; + let build_keys = K::build_many(&self.build_key_idxs, build_chunk); for (build_row_id, (build_key, visible)) in build_keys .into_iter() @@ -354,7 +354,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -417,7 +417,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -486,7 +486,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -564,7 +564,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -628,7 +628,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -703,7 +703,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -785,7 +785,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -850,7 +850,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -922,7 +922,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_key, visible) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -976,7 +976,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -1049,7 +1049,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) @@ -1131,7 +1131,7 @@ impl HashJoinExecutor { #[for_await] for probe_chunk in probe_side.execute() { let probe_chunk = probe_chunk?; - let probe_keys = K::build(&probe_key_idxs, &probe_chunk)?; + let probe_keys = K::build_many(&probe_key_idxs, &probe_chunk); for (probe_row_id, (probe_key, visible)) in probe_keys .iter() .zip_eq_fast(probe_chunk.visibility().iter()) diff --git a/src/batch/src/executor/join/local_lookup_join.rs b/src/batch/src/executor/join/local_lookup_join.rs index 85762d5831a60..17b257106fb5b 100644 --- a/src/batch/src/executor/join/local_lookup_join.rs +++ b/src/batch/src/executor/join/local_lookup_join.rs @@ -19,6 +19,7 @@ use anyhow::Context; use itertools::Itertools; use risingwave_common::buffer::BitmapBuilder; use risingwave_common::catalog::{ColumnDesc, Field, Schema}; +use risingwave_common::hash::table_distribution::TableDistribution; use risingwave_common::hash::{ ExpandedParallelUnitMapping, HashKey, HashKeyDispatcher, ParallelUnitId, VirtualNode, }; @@ -50,6 +51,7 @@ use crate::task::{BatchTaskContext, ShutdownToken, TaskId}; /// Inner side executor builder for the `LocalLookupJoinExecutor` struct InnerSideExecutorBuilder { table_desc: StorageTableDesc, + table_distribution: TableDistribution, vnode_mapping: ExpandedParallelUnitMapping, outer_side_key_types: Vec, inner_side_schema: Schema, @@ -81,15 +83,8 @@ pub type BoxedLookupExecutorBuilder = Box; impl InnerSideExecutorBuilder { /// Gets the virtual node based on the given `scan_range` fn get_virtual_node(&self, scan_range: &ScanRange) -> Result { - let dist_key_in_pk_indices = self - .table_desc - .dist_key_in_pk_indices - .iter() - .map(|&k| k as usize) - .collect_vec(); - let virtual_node = scan_range - .try_compute_vnode_with_dist_key_in_pk_indices(&dist_key_in_pk_indices) + .try_compute_vnode(&self.table_distribution) .context("Could not compute vnode for lookup join")?; Ok(virtual_node) } @@ -224,6 +219,7 @@ impl LookupExecutorBuilder for InnerSideExecutorBuilder let exchange_node = NodeBody::Exchange(ExchangeNode { sources, + sequential: true, input_schema: self.inner_side_schema.to_prost(), }); @@ -379,6 +375,10 @@ impl BoxedExecutorBuilder for LocalLookupJoinExecutorBuilder { let inner_side_builder = InnerSideExecutorBuilder { table_desc: table_desc.clone(), + table_distribution: TableDistribution::new_from_storage_table_desc( + Some(TableDistribution::all_vnodes()), + table_desc, + ), vnode_mapping, outer_side_key_types, inner_side_schema, diff --git a/src/batch/src/executor/join/lookup_join_base.rs b/src/batch/src/executor/join/lookup_join_base.rs index 6ff1492f06f33..8128e901ab7a5 100644 --- a/src/batch/src/executor/join/lookup_join_base.rs +++ b/src/batch/src/executor/join/lookup_join_base.rs @@ -146,7 +146,7 @@ impl LookupJoinBase { // Build hash map for (build_chunk_id, build_chunk) in build_side.iter().enumerate() { - let build_keys = K::build(&hash_join_build_side_key_idxs, build_chunk)?; + let build_keys = K::build_many(&hash_join_build_side_key_idxs, build_chunk); for (build_row_id, (build_key, visible)) in build_keys .into_iter() diff --git a/src/batch/src/executor/mod.rs b/src/batch/src/executor/mod.rs index 3b0298643484a..ded2af7089826 100644 --- a/src/batch/src/executor/mod.rs +++ b/src/batch/src/executor/mod.rs @@ -20,6 +20,7 @@ mod generic_exchange; mod group_top_n; mod hash_agg; mod hop_window; +mod iceberg_scan; mod insert; mod join; mod limit; @@ -52,6 +53,7 @@ pub use generic_exchange::*; pub use group_top_n::*; pub use hash_agg::*; pub use hop_window::*; +pub use iceberg_scan::*; pub use insert::*; pub use join::*; pub use limit::*; diff --git a/src/batch/src/executor/project_set.rs b/src/batch/src/executor/project_set.rs index 97ac9b9d88cfd..b7291609be586 100644 --- a/src/batch/src/executor/project_set.rs +++ b/src/batch/src/executor/project_set.rs @@ -230,7 +230,7 @@ mod tests { let fields = &proj_executor.schema().fields; assert_eq!(fields[0].data_type, DataType::Int32); - let expected = vec![DataChunk::from_pretty( + let expected = [DataChunk::from_pretty( "I i i i 0 1 1 2 1 1 1 2 diff --git a/src/batch/src/executor/row_seq_scan.rs b/src/batch/src/executor/row_seq_scan.rs index 6a5ec3cdf704f..bf2fb9613b7eb 100644 --- a/src/batch/src/executor/row_seq_scan.rs +++ b/src/batch/src/executor/row_seq_scan.rs @@ -24,7 +24,6 @@ use risingwave_common::catalog::{ColumnId, Schema}; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, Datum}; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; -use risingwave_common::util::select_all; use risingwave_common::util::value_encoding::deserialize_datum; use risingwave_pb::batch_plan::plan_node::NodeBody; use risingwave_pb::batch_plan::{scan_range, PbScanRange}; @@ -34,6 +33,7 @@ use risingwave_storage::store::PrefetchOptions; use risingwave_storage::table::batch_table::storage_table::StorageTable; use risingwave_storage::table::{collect_data_chunk, TableDistribution}; use risingwave_storage::{dispatch_state_store, StateStore}; +use rw_futures_util::select_all; use crate::error::{BatchError, Result}; use crate::executor::{ diff --git a/src/batch/src/executor/source.rs b/src/batch/src/executor/source.rs index 2714d5335b906..4a40a45a6c28a 100644 --- a/src/batch/src/executor/source.rs +++ b/src/batch/src/executor/source.rs @@ -17,30 +17,34 @@ use std::sync::Arc; use futures::StreamExt; use futures_async_stream::try_stream; +use itertools::Itertools; use risingwave_common::array::{DataChunk, Op, StreamChunk}; use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableId}; use risingwave_common::types::DataType; use risingwave_connector::parser::SpecificParserConfig; +use risingwave_connector::source::iceberg::{IcebergProperties, IcebergSplit}; use risingwave_connector::source::monitor::SourceMetrics; +use risingwave_connector::source::reader::reader::SourceReader; use risingwave_connector::source::{ ConnectorProperties, SourceColumnDesc, SourceContext, SourceCtrlOpts, SplitImpl, SplitMetaData, }; use risingwave_pb::batch_plan::plan_node::NodeBody; -use risingwave_source::connector_source::ConnectorSource; use super::Executor; use crate::error::{BatchError, Result}; -use crate::executor::{BoxedExecutor, BoxedExecutorBuilder, ExecutorBuilder}; +use crate::executor::{ + BoxedExecutor, BoxedExecutorBuilder, ExecutorBuilder, FileSelector, IcebergScanExecutor, +}; use crate::task::BatchTaskContext; pub struct SourceExecutor { - connector_source: ConnectorSource, + source: SourceReader, // used to create reader column_ids: Vec, metrics: Arc, source_id: TableId, - split: SplitImpl, + split_list: Vec, schema: Schema, identity: String, @@ -75,18 +79,9 @@ impl BoxedExecutorBuilder for SourceExecutor { .map(|c| SourceColumnDesc::from(&ColumnDesc::from(c.column_desc.as_ref().unwrap()))) .collect(); - let connector_source = ConnectorSource { - config, - columns, - parser_config, - connector_message_buffer_size: source - .context() - .get_config() - .developer - .connector_message_buffer_size, - }; let source_ctrl_opts = SourceCtrlOpts { chunk_size: source.context().get_config().developer.chunk_size, + rate_limit: None, }; let column_ids: Vec<_> = source_node @@ -95,7 +90,11 @@ impl BoxedExecutorBuilder for SourceExecutor { .map(|column| ColumnId::from(column.get_column_desc().unwrap().column_id)) .collect(); - let split = SplitImpl::restore_from_bytes(&source_node.split)?; + let split_list = source_node + .split + .iter() + .map(|split| SplitImpl::restore_from_bytes(split).unwrap()) + .collect_vec(); let fields = source_node .columns @@ -109,16 +108,45 @@ impl BoxedExecutorBuilder for SourceExecutor { .collect(); let schema = Schema::new(fields); - Ok(Box::new(SourceExecutor { - connector_source, - column_ids, - metrics: source.context().source_metrics(), - source_id: TableId::new(source_node.source_id), - split, - schema, - identity: source.plan_node().get_identity().clone(), - source_ctrl_opts, - })) + if let ConnectorProperties::Iceberg(iceberg_properties) = config { + let iceberg_properties: IcebergProperties = *iceberg_properties; + assert_eq!(split_list.len(), 1); + if let SplitImpl::Iceberg(split) = &split_list[0] { + let split: IcebergSplit = split.clone(); + Ok(Box::new(IcebergScanExecutor::new( + iceberg_properties.to_iceberg_config(), + Some(split.snapshot_id), + FileSelector::FileList(split.files), + source.context.get_config().developer.chunk_size, + schema, + source.plan_node().get_identity().clone(), + ))) + } else { + unreachable!() + } + } else { + let source_reader = SourceReader { + config, + columns, + parser_config, + connector_message_buffer_size: source + .context() + .get_config() + .developer + .connector_message_buffer_size, + }; + + Ok(Box::new(SourceExecutor { + source: source_reader, + column_ids, + metrics: source.context().source_metrics(), + source_id: TableId::new(source_node.source_id), + split_list, + schema, + identity: source.plan_node().get_identity().clone(), + source_ctrl_opts, + })) + } } } @@ -147,16 +175,17 @@ impl SourceExecutor { self.source_ctrl_opts.clone(), None, ConnectorProperties::default(), + "NA".to_owned(), // FIXME: source name was not passed in batch plan )); let stream = self - .connector_source - .stream_reader(Some(vec![self.split]), self.column_ids, source_ctx) + .source + .to_stream(Some(self.split_list), self.column_ids, source_ctx) .await?; #[for_await] for chunk in stream { let chunk = chunk.map_err(BatchError::connector)?; - let data_chunk = covert_stream_chunk_to_batch_chunk(chunk.chunk)?; + let data_chunk = covert_stream_chunk_to_batch_chunk(chunk)?; if data_chunk.capacity() > 0 { yield data_chunk; } diff --git a/src/batch/src/executor/sys_row_seq_scan.rs b/src/batch/src/executor/sys_row_seq_scan.rs index 6bebf862e5ea8..fd427b132909f 100644 --- a/src/batch/src/executor/sys_row_seq_scan.rs +++ b/src/batch/src/executor/sys_row_seq_scan.rs @@ -15,9 +15,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::DataChunk; -use risingwave_common::catalog::{ColumnDesc, ColumnId, Schema, SysCatalogReaderRef, TableId}; -use risingwave_common::row::{OwnedRow, Row}; -use risingwave_common::types::ToOwnedDatum; +use risingwave_common::catalog::{ColumnDesc, Schema, SysCatalogReaderRef, TableId}; use risingwave_pb::batch_plan::plan_node::NodeBody; use crate::error::{BatchError, Result}; @@ -29,7 +27,7 @@ use crate::task::BatchTaskContext; pub struct SysRowSeqScanExecutor { table_id: TableId, schema: Schema, - column_ids: Vec, + column_indices: Vec, identity: String, sys_catalog_reader: SysCatalogReaderRef, @@ -39,14 +37,14 @@ impl SysRowSeqScanExecutor { pub fn new( table_id: TableId, schema: Schema, - column_id: Vec, + column_indices: Vec, identity: String, sys_catalog_reader: SysCatalogReaderRef, ) -> Self { Self { table_id, schema, - column_ids: column_id, + column_indices, identity, sys_catalog_reader, } @@ -78,12 +76,15 @@ impl BoxedExecutorBuilder for SysRowSeqScanExecutorBuilder { .map(|column_desc| ColumnDesc::from(column_desc.clone())) .collect_vec(); - let column_ids = column_descs.iter().map(|d| d.column_id).collect_vec(); + let column_indices = column_descs + .iter() + .map(|d| d.column_id.get_id() as usize) + .collect_vec(); let schema = Schema::new(column_descs.iter().map(Into::into).collect_vec()); Ok(Box::new(SysRowSeqScanExecutor::new( table_id, schema, - column_ids, + column_indices, source.plan_node().get_identity().clone(), sys_catalog_reader, ))) @@ -107,26 +108,13 @@ impl Executor for SysRowSeqScanExecutor { impl SysRowSeqScanExecutor { #[try_stream(boxed, ok = DataChunk, error = BatchError)] async fn do_executor(self: Box) { - let rows = self + let chunk = self .sys_catalog_reader .read_table(&self.table_id) .await .map_err(BatchError::SystemTable)?; - let filtered_rows = rows - .iter() - .map(|row| { - let datums = self - .column_ids - .iter() - .map(|column_id| row.datum_at(column_id.get_id() as usize).to_owned_datum()) - .collect_vec(); - OwnedRow::new(datums) - }) - .collect_vec(); - - if !filtered_rows.is_empty() { - let chunk = DataChunk::from_rows(&filtered_rows, &self.schema.data_types()); - yield chunk + if chunk.cardinality() != 0 { + yield chunk.project(&self.column_indices); } } } diff --git a/src/batch/src/executor/union.rs b/src/batch/src/executor/union.rs index 00d01f93448f7..e37baed08debc 100644 --- a/src/batch/src/executor/union.rs +++ b/src/batch/src/executor/union.rs @@ -17,8 +17,8 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::DataChunk; use risingwave_common::catalog::Schema; -use risingwave_common::util::select_all; use risingwave_pb::batch_plan::plan_node::NodeBody; +use rw_futures_util::select_all; use crate::error::{BatchError, Result}; use crate::executor::{ diff --git a/src/batch/src/executor/update.rs b/src/batch/src/executor/update.rs index 1706a5f5cba7a..9e9c2fa8f543d 100644 --- a/src/batch/src/executor/update.rs +++ b/src/batch/src/executor/update.rs @@ -22,9 +22,9 @@ use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::types::DataType; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; use risingwave_common::util::iter_util::ZipEqDebug; +use risingwave_dml::dml_manager::DmlManagerRef; use risingwave_expr::expr::{build_from_prost, BoxedExpression}; use risingwave_pb::batch_plan::plan_node::NodeBody; -use risingwave_source::dml_manager::DmlManagerRef; use crate::error::{BatchError, Result}; use crate::executor::{ @@ -136,7 +136,7 @@ impl UpdateExecutor { let mut builder = DataChunkBuilder::new(data_types, self.chunk_size); - let mut write_handle: risingwave_source::WriteHandle = + let mut write_handle: risingwave_dml::WriteHandle = table_dml_handle.write_handle(self.session_id, self.txn_id)?; write_handle.begin()?; @@ -264,8 +264,8 @@ mod tests { schema_test_utils, ColumnDesc, ColumnId, INITIAL_TABLE_VERSION_ID, }; use risingwave_common::test_prelude::DataChunkTestExt; + use risingwave_dml::dml_manager::DmlManager; use risingwave_expr::expr::InputRefExpression; - use risingwave_source::dml_manager::DmlManager; use super::*; use crate::executor::test_utils::MockExecutor; diff --git a/src/batch/src/task/context.rs b/src/batch/src/task/context.rs index 6ad50581dc01f..70fab761e6569 100644 --- a/src/batch/src/task/context.rs +++ b/src/batch/src/task/context.rs @@ -19,8 +19,8 @@ use risingwave_common::config::BatchConfig; use risingwave_common::memory::MemoryContext; use risingwave_common::util::addr::{is_local_address, HostAddr}; use risingwave_connector::source::monitor::SourceMetrics; +use risingwave_dml::dml_manager::DmlManagerRef; use risingwave_rpc_client::ComputeClientPoolRef; -use risingwave_source::dml_manager::DmlManagerRef; use risingwave_storage::StateStoreImpl; use super::TaskId; diff --git a/src/batch/src/task/env.rs b/src/batch/src/task/env.rs index 66fb433d6ca8b..2099adad4da65 100644 --- a/src/batch/src/task/env.rs +++ b/src/batch/src/task/env.rs @@ -18,8 +18,8 @@ use risingwave_common::config::BatchConfig; use risingwave_common::util::addr::HostAddr; use risingwave_common::util::worker_util::WorkerNodeId; use risingwave_connector::source::monitor::SourceMetrics; +use risingwave_dml::dml_manager::DmlManagerRef; use risingwave_rpc_client::ComputeClientPoolRef; -use risingwave_source::dml_manager::DmlManagerRef; use risingwave_storage::StateStoreImpl; use crate::monitor::{BatchExecutorMetrics, BatchManagerMetrics, BatchTaskMetrics}; @@ -91,8 +91,8 @@ impl BatchEnvironment { // Create an instance for testing purpose. #[cfg(test)] pub fn for_test() -> Self { + use risingwave_dml::dml_manager::DmlManager; use risingwave_rpc_client::ComputeClientPool; - use risingwave_source::dml_manager::DmlManager; use risingwave_storage::monitor::MonitoredStorageMetrics; BatchEnvironment { diff --git a/src/bench/Cargo.toml b/src/bench/Cargo.toml index e5f6d8e7ede1a..2aa378d857f83 100644 --- a/src/bench/Cargo.toml +++ b/src/bench/Cargo.toml @@ -8,6 +8,7 @@ license = { workspace = true } repository = { workspace = true } [dependencies] +anyhow = "1" async-trait = "0.1" aws-config = { workspace = true } aws-sdk-s3 = { workspace = true } @@ -18,17 +19,23 @@ bytes = "1" bytesize = { version = "1", features = ["serde"] } clap = { version = "4", features = ["derive"] } futures = { version = "0.3", default-features = false, features = ["alloc"] } +futures-async-stream = { workspace = true } hdrhistogram = "7" itertools = "0.12" libc = "0.2" -opentelemetry = { version = "0.20", default-features = false, features = ["rt-tokio"], optional = true } +opentelemetry = { workspace = true, optional = true } parking_lot = "0.12" prometheus = { version = "0.13", features = ["process"] } rand = "0.8" risingwave_common = { workspace = true } +risingwave_connector = { workspace = true } +risingwave_pb = { workspace = true } risingwave_rt = { workspace = true, optional = true } risingwave_storage = { workspace = true } +risingwave_stream = { workspace = true } serde = { version = "1", features = ["derive"] } +serde_yaml = "0.9" +thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ "fs", "rt", @@ -47,12 +54,16 @@ tracing-subscriber = "0.3.17" workspace-hack = { path = "../workspace-hack" } [target.'cfg(target_os = "linux")'.dependencies] -nix = { version = "0.27", features = ["fs", "mman"] } +nix = { version = "0.28", features = ["fs", "mman"] } [[bin]] name = "s3-bench" path = "s3_bench/main.rs" +[[bin]] +name = "sink-bench" +path = "sink_bench/main.rs" + [features] bpf = ["bcc", "risingwave_storage/bpf"] trace = ["opentelemetry", "risingwave_rt", "tracing/release_max_level_trace"] diff --git a/src/bench/s3_bench/main.rs b/src/bench/s3_bench/main.rs index 8eec3e6cfbeea..792c9c4743dbf 100644 --- a/src/bench/s3_bench/main.rs +++ b/src/bench/s3_bench/main.rs @@ -28,7 +28,6 @@ use futures::stream::{self, StreamExt}; use futures::{future, Future, FutureExt}; use itertools::Itertools; use rand::{Rng, SeedableRng}; -use risingwave_common::error::RwError; use tokio::join; use tokio::sync::RwLock; use tracing::debug; @@ -233,7 +232,7 @@ async fn multi_part_upload( let part_t = Instant::now(); let result = a.send().await.unwrap(); let part_ttl = part_t.elapsed(); - Ok::<_, RwError>((result, part_ttl)) + Ok::<_, anyhow::Error>((result, part_ttl)) }) .collect_vec(); let ttfb = t.elapsed(); @@ -318,7 +317,7 @@ async fn multi_part_get( .into_iter() .map(create_part_get) .map(|resp| async move { - let result: Result<(usize, Duration), RwError> = Ok(( + let result: anyhow::Result<(usize, Duration)> = Ok(( resp.await .unwrap() .body @@ -381,7 +380,7 @@ async fn run_case( cfg: Arc, client: Arc, objs: Arc>, -) -> Result<(), RwError> { +) -> anyhow::Result<()> { let (name, analysis) = match case.clone() { Case::Put { name, diff --git a/src/bench/sink_bench/main.rs b/src/bench/sink_bench/main.rs new file mode 100644 index 0000000000000..66572ca67cdf2 --- /dev/null +++ b/src/bench/sink_bench/main.rs @@ -0,0 +1,484 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#![feature(coroutines)] +#![feature(proc_macro_hygiene)] +#![feature(stmt_expr_attributes)] +#![feature(let_chains)] + +use core::str::FromStr; +use std::collections::HashMap; + +use anyhow::anyhow; +use clap::Parser; +use futures::prelude::future::Either; +use futures::prelude::stream::{BoxStream, PollNext}; +use futures::stream::select_with_strategy; +use futures::{FutureExt, StreamExt, TryStreamExt}; +use futures_async_stream::try_stream; +use risingwave_common::buffer::Bitmap; +use risingwave_common::catalog::ColumnId; +use risingwave_connector::dispatch_sink; +use risingwave_connector::parser::{ + EncodingProperties, ParserConfig, ProtocolProperties, SpecificParserConfig, +}; +use risingwave_connector::sink::catalog::{ + SinkEncode, SinkFormat, SinkFormatDesc, SinkId, SinkType, +}; +use risingwave_connector::sink::log_store::{ + LogReader, LogStoreReadItem, LogStoreResult, TruncateOffset, +}; +use risingwave_connector::sink::mock_coordination_client::MockMetaClient; +use risingwave_connector::sink::{ + build_sink, LogSinker, Sink, SinkError, SinkMetaClient, SinkParam, SinkWriterParam, + SINK_TYPE_APPEND_ONLY, SINK_TYPE_UPSERT, +}; +use risingwave_connector::source::datagen::{ + DatagenProperties, DatagenSplitEnumerator, DatagenSplitReader, +}; +use risingwave_connector::source::{Column, DataType, SplitEnumerator, SplitReader}; +use risingwave_pb::connector_service::SinkPayloadFormat; +use risingwave_stream::executor::test_utils::prelude::ColumnDesc; +use risingwave_stream::executor::{Barrier, Message, MessageStreamItem, StreamExecutorError}; +use serde::{Deserialize, Deserializer}; +use thiserror_ext::AsReport; +use tokio::sync::oneshot::Sender; +use tokio::time::{sleep, Instant}; + +const CHECKPOINT_INTERVAL: u64 = 1000; +const THROUGHPUT_METRIC_RECORD_INTERVAL: u128 = 500; +const BENCH_TIME: u64 = 20; +const BENCH_TEST: &str = "bench_test"; + +pub struct MockRangeLogReader { + upstreams: BoxStream<'static, MessageStreamItem>, + current_epoch: u64, + chunk_id: usize, + throughput_metric: Option, + stop_rx: tokio::sync::mpsc::Receiver<()>, + result_tx: Option>, +} + +impl LogReader for MockRangeLogReader { + async fn init(&mut self) -> LogStoreResult<()> { + self.throughput_metric.as_mut().unwrap().add_metric(0); + Ok(()) + } + + async fn next_item(&mut self) -> LogStoreResult<(u64, LogStoreReadItem)> { + tokio::select! { + _ = self.stop_rx.recv() => { + self.result_tx + .take() + .unwrap() + .send(self.throughput_metric.take().unwrap()) + .map_err(|_| anyhow!("Can't send throughput_metric"))?; + futures::future::pending().await + }, + item = self.upstreams.next() => { + match item.unwrap().unwrap() { + Message::Barrier(barrier) => { + let prev_epoch = self.current_epoch; + self.current_epoch = barrier.epoch.curr; + Ok(( + prev_epoch, + LogStoreReadItem::Barrier { + is_checkpoint: true, + }, + )) + } + Message::Chunk(chunk) => { + self.throughput_metric.as_mut().unwrap().add_metric(chunk.capacity()); + self.chunk_id += 1; + Ok(( + self.current_epoch, + LogStoreReadItem::StreamChunk { + chunk, + chunk_id: self.chunk_id, + }, + )) + } + _ => Err(anyhow!("Can't assert message type".to_string())), + } + } + } + } + + async fn truncate(&mut self, _offset: TruncateOffset) -> LogStoreResult<()> { + Ok(()) + } + + async fn rewind(&mut self) -> LogStoreResult<(bool, Option)> { + Ok((false, None)) + } +} + +impl MockRangeLogReader { + fn new( + mock_source: MockDatagenSource, + throughput_metric: ThroughputMetric, + stop_rx: tokio::sync::mpsc::Receiver<()>, + result_tx: Sender, + ) -> MockRangeLogReader { + MockRangeLogReader { + upstreams: mock_source.into_stream().boxed(), + current_epoch: 0, + chunk_id: 0, + throughput_metric: Some(throughput_metric), + stop_rx, + result_tx: Some(result_tx), + } + } +} + +struct ThroughputMetric { + chunk_size_list: Vec<(u64, Instant)>, + accumulate_chunk_size: u64, + // Record every `record_interval` ms + record_interval: u128, + last_record_time: Instant, +} + +impl ThroughputMetric { + pub fn new() -> Self { + Self { + chunk_size_list: vec![], + accumulate_chunk_size: 0, + record_interval: THROUGHPUT_METRIC_RECORD_INTERVAL, + last_record_time: Instant::now(), + } + } + + pub fn add_metric(&mut self, chunk_size: usize) { + self.accumulate_chunk_size += chunk_size as u64; + if Instant::now() + .duration_since(self.last_record_time) + .as_millis() + > self.record_interval + { + self.chunk_size_list + .push((self.accumulate_chunk_size, Instant::now())); + self.last_record_time = Instant::now(); + } + } + + pub fn get_throughput(&self) -> Vec { + #[allow(clippy::disallowed_methods)] + self.chunk_size_list + .iter() + .zip(self.chunk_size_list.iter().skip(1)) + .map(|(current, next)| { + let throughput = (next.0 - current.0) * 1000 + / (next.1.duration_since(current.1).as_millis() as u64); + format!("{} rows/s", throughput) + }) + .collect() + } +} + +pub struct MockDatagenSource { + datagen_split_readers: Vec, +} +impl MockDatagenSource { + pub async fn new( + rows_per_second: u64, + source_schema: Vec, + split_num: String, + ) -> MockDatagenSource { + let properties = DatagenProperties { + split_num: Some(split_num), + rows_per_second, + fields: HashMap::default(), + }; + let mut datagen_enumerator = + DatagenSplitEnumerator::new(properties.clone(), Default::default()) + .await + .unwrap(); + let parser_config = ParserConfig { + specific: SpecificParserConfig { + key_encoding_config: None, + encoding_config: EncodingProperties::Native, + protocol_config: ProtocolProperties::Native, + }, + ..Default::default() + }; + let mut datagen_split_readers = vec![]; + let mut datagen_splits = datagen_enumerator.list_splits().await.unwrap(); + while let Some(splits) = datagen_splits.pop() { + datagen_split_readers.push( + DatagenSplitReader::new( + properties.clone(), + vec![splits], + parser_config.clone(), + Default::default(), + Some(source_schema.clone()), + ) + .await + .unwrap(), + ); + } + MockDatagenSource { + datagen_split_readers, + } + } + + #[try_stream(ok = Message, error = StreamExecutorError)] + pub async fn source_to_data_stream(mut self) { + let mut readers = vec![]; + while let Some(reader) = self.datagen_split_readers.pop() { + readers.push(reader.into_stream()); + } + loop { + for i in &mut readers { + let item = i.next().await.unwrap().unwrap(); + yield Message::Chunk(item); + } + } + } + + #[try_stream(ok = Message, error = StreamExecutorError)] + pub async fn into_stream(self) { + let stream = select_with_strategy( + Self::barrier_to_message_stream().map_ok(Either::Left), + self.source_to_data_stream().map_ok(Either::Right), + |_: &mut PollNext| PollNext::Left, + ); + #[for_await] + for message in stream { + match message.unwrap() { + Either::Left(Message::Barrier(barrier)) => { + yield Message::Barrier(barrier); + } + Either::Right(Message::Chunk(chunk)) => yield Message::Chunk(chunk), + _ => { + return Err(StreamExecutorError::from( + "Can't assert message type".to_string(), + )) + } + } + } + } + + #[try_stream(ok = Message, error = StreamExecutorError)] + pub async fn barrier_to_message_stream() { + let mut epoch = 0_u64; + loop { + let prev_epoch = epoch; + epoch += 1; + let barrier = Barrier::with_prev_epoch_for_test(epoch, prev_epoch); + yield Message::Barrier(barrier); + sleep(tokio::time::Duration::from_millis(CHECKPOINT_INTERVAL)).await; + } + } +} + +async fn consume_log_stream( + sink: S, + mut log_reader: MockRangeLogReader, + mut sink_writer_param: SinkWriterParam, +) -> Result<(), String> +where + ::Coordinator: std::marker::Send, + ::Coordinator: 'static, +{ + if let Ok(coordinator) = sink.new_coordinator().await { + sink_writer_param.meta_client = Some(SinkMetaClient::MockMetaClient(MockMetaClient::new( + Box::new(coordinator), + ))); + sink_writer_param.vnode_bitmap = Some(Bitmap::ones(1)); + } + let log_sinker = sink.new_log_sinker(sink_writer_param).await.unwrap(); + if let Err(e) = log_sinker.consume_log_and_sink(&mut log_reader).await { + return Err(e.to_report_string()); + } + Err("Stream closed".to_string()) +} + +#[derive(Debug, Deserialize)] +#[allow(dead_code)] +struct TableSchemaFromYml { + table_name: String, + pk_indices: Vec, + columns: Vec, +} + +impl TableSchemaFromYml { + pub fn get_source_schema(&self) -> Vec { + self.columns + .iter() + .map(|column| Column { + name: column.name.clone(), + data_type: column.r#type.clone(), + is_visible: true, + }) + .collect() + } + + pub fn get_sink_schema(&self) -> Vec { + self.columns + .iter() + .map(|column| { + ColumnDesc::named(column.name.clone(), ColumnId::new(1), column.r#type.clone()) + }) + .collect() + } +} +#[derive(Debug, Deserialize)] +struct ColumnDescFromYml { + name: String, + #[serde(deserialize_with = "deserialize_datatype")] + r#type: DataType, +} + +fn deserialize_datatype<'de, D>(deserializer: D) -> Result +where + D: Deserializer<'de>, +{ + let s: &str = Deserialize::deserialize(deserializer)?; + DataType::from_str(s).map_err(serde::de::Error::custom) +} + +fn read_table_schema_from_yml(path: &str) -> TableSchemaFromYml { + let data = std::fs::read_to_string(path).unwrap(); + let table: TableSchemaFromYml = serde_yaml::from_str(&data).unwrap(); + table +} + +fn read_sink_option_from_yml(path: &str) -> HashMap> { + let data = std::fs::read_to_string(path).unwrap(); + let sink_option: HashMap> = + serde_yaml::from_str(&data).unwrap(); + sink_option +} + +#[derive(Parser, Debug)] +pub struct Config { + #[clap(long, default_value = "./sink_bench/schema.yml")] + schema_path: String, + + #[clap(short, long, default_value = "./sink_bench/sink_option.yml")] + option_path: String, + + #[clap(short, long, default_value = BENCH_TEST)] + sink: String, + + #[clap(short, long)] + rows_per_second: u64, + + #[clap(long, default_value = "10")] + split_num: String, +} + +fn mock_from_legacy_type( + connector: &str, + r#type: &str, +) -> Result, SinkError> { + use risingwave_connector::sink::redis::RedisSink; + use risingwave_connector::sink::Sink as _; + if connector.eq(RedisSink::SINK_NAME) { + let format = match r#type { + SINK_TYPE_APPEND_ONLY => SinkFormat::AppendOnly, + SINK_TYPE_UPSERT => SinkFormat::Upsert, + _ => { + return Err(SinkError::Config(anyhow!( + "sink type unsupported: {}", + r#type + ))) + } + }; + Ok(Some(SinkFormatDesc { + format, + encode: SinkEncode::Json, + options: Default::default(), + })) + } else { + SinkFormatDesc::from_legacy_type(connector, r#type) + } +} + +fn print_throughput_result(throughput_metric: ThroughputMetric) { + let throughput_result = throughput_metric.get_throughput(); + if throughput_result.is_empty() { + println!("Throughput Sink: Don't get Throughput, please check"); + } else { + println!("Throughput Sink: {:?}", throughput_result); + } +} + +#[tokio::main] +async fn main() { + let cfg = Config::parse(); + let table_schema = read_table_schema_from_yml(&cfg.schema_path); + let mock_datagen_source = MockDatagenSource::new( + cfg.rows_per_second, + table_schema.get_source_schema(), + cfg.split_num, + ) + .await; + let (data_size_tx, data_size_rx) = tokio::sync::oneshot::channel::(); + let (stop_tx, stop_rx) = tokio::sync::mpsc::channel::<()>(5); + let throughput_metric = ThroughputMetric::new(); + + let mut mock_range_log_reader = MockRangeLogReader::new( + mock_datagen_source, + throughput_metric, + stop_rx, + data_size_tx, + ); + if cfg.sink.eq(&BENCH_TEST.to_string()) { + println!("Start Sink Bench!, Wait {:?}s", BENCH_TIME); + tokio::spawn(async move { + mock_range_log_reader.init().await.unwrap(); + loop { + mock_range_log_reader.next_item().await.unwrap(); + } + }); + } else { + let properties = read_sink_option_from_yml(&cfg.option_path) + .get(&cfg.sink) + .expect("Sink type error") + .clone(); + + let connector = properties.get("connector").unwrap().clone(); + let format_desc = mock_from_legacy_type( + &connector.clone(), + properties.get("type").unwrap_or(&"append-only".to_string()), + ) + .unwrap(); + let sink_param = SinkParam { + sink_id: SinkId::new(1), + properties, + columns: table_schema.get_sink_schema(), + downstream_pk: table_schema.pk_indices, + sink_type: SinkType::AppendOnly, + format_desc, + db_name: "not_need_set".to_string(), + sink_from_name: "not_need_set".to_string(), + }; + let sink = build_sink(sink_param).unwrap(); + let mut sink_writer_param = SinkWriterParam::for_test(); + println!("Start Sink Bench!, Wait {:?}s", BENCH_TIME); + sink_writer_param.connector_params.sink_payload_format = SinkPayloadFormat::StreamChunk; + tokio::spawn(async move { + dispatch_sink!(sink, sink, { + consume_log_stream(sink, mock_range_log_reader, sink_writer_param).boxed() + }) + .await + .unwrap(); + }); + } + sleep(tokio::time::Duration::from_secs(BENCH_TIME)).await; + println!("Bench Over!"); + stop_tx.send(()).await.unwrap(); + print_throughput_result(data_size_rx.await.unwrap()); +} diff --git a/src/bench/sink_bench/schema.yml b/src/bench/sink_bench/schema.yml new file mode 100644 index 0000000000000..e339dec5a351b --- /dev/null +++ b/src/bench/sink_bench/schema.yml @@ -0,0 +1,12 @@ +table_name: "sink_bench" +pk_indices: + - 0 +columns: + - name: "v1" + type: "int" + - name: "v2" + type: "bigint" + - name: "v3" + type: "varchar" + - name: "v4" + type: "varchar" \ No newline at end of file diff --git a/src/bench/sink_bench/sink_option.yml b/src/bench/sink_bench/sink_option.yml new file mode 100644 index 0000000000000..3a942db4edb30 --- /dev/null +++ b/src/bench/sink_bench/sink_option.yml @@ -0,0 +1,107 @@ +Clickhouse: + connector: "clickhouse" + type: "append-only" + force_append_only: "true" + clickhouse.url: "http://127.0.0.1:8123" + clickhouse.user: "default" + clickhouse.password: "" + clickhouse.database: "default" + clickhouse.table: "sink_bench" +Redis: + connector: 'redis' + primary_key: 'v1' + type: 'append-only' + redis.url: 'redis://127.0.0.1:6379/' +Kafka: + connector: 'kafka' + properties.bootstrap.server: '127.0.0.1:9092' + topic : 'counts' + primary_key: 'v1' +Pulsar: + connector: 'pulsar' + pulsar.topic: 'twitter' + pulsar.service.url: 'pulsar://127.0.0.1:6650' +Iceberg: + connector: 'iceberg' + type: 'append-only' + warehouse.path: 's3a://iceberg-data' + s3.endpoint: 'http://127.0.0.1:9301' + s3.access.key: 'hummockadmin' + s3.secret.key: 'hummockadmin' + database.name: 'demo_db' + s3.region: '1' + table.name: 'demo_db.demo_table' + force_append_only: 'true' +# RemoteIceberg: +# connector: 'iceberg_java' +# type: 'append-only' +# force_append_only: 'true' +# warehouse.path: 's3://iceberg-data' +# s3.endpoint: 'http://127.0.0.1:9301' +# s3.access.key: 'hummockadmin' +# s3.secret.key: 'hummockadmin' +# database.name: 'demo_db' +# table.name: 'demo_table' +Mysql: + connector: 'jdbc' + type: 'append-only' + force_append_only: 'true' + jdbc.url: 'jdbc:mysql://127.0.0.1:3306/mydb?user=username&password=123456' + table.name: 'bench_table' +Postgres: + connector: 'jdbc' + type: 'append-only' + force_append_only: 'true' + jdbc.url: 'jdbc:postgresql://127.0.0.1:5432/mydb?user=postgres&password=123' + table.name: 'bench_table' +DeltaLake: + connector: 'deltalake' + type: 'append-only' + force_append_only: 'true' + location: 's3a://deltalake-bench/deltalake' + s3.access.key: 'hummockadmin' + s3.secret.key: 'hummockadmin' + s3.endpoint: 'http://127.0.0.1:9301' +ElasticSearch: + connector: 'elasticsearch' + index: 'test' + url: 'http://127.0.0.1:9200' + username: 'elastic' + password: 'risingwave' + delimiter: '_' +Cassandra: + connector: 'cassandra' + type: 'append-only' + force_append_only: 'true' + cassandra.url: '127.0.0.1:9042' + cassandra.keyspace: 'demo' + cassandra.table: 'table_bench' + cassandra.datacenter: 'datacenter1' +Doris: + connector: 'doris' + type: 'append-only' + doris.url: 'http://127.0.0.1:8030' + doris.user: 'users' + doris.password: '123456' + doris.database: 'demo' + doris.table: 'table_bench' + force_append_only: 'true' +Starrocks: + connector: 'starrocks' + type: 'append-only' + starrocks.host: '127.0.0.1' + starrocks.mysqlport: '9030' + starrocks.httpport: '8030' + starrocks.user: 'users' + starrocks.password: '123456' + starrocks.database: 'demo' + starrocks.table: 'table_bench' + force_append_only: 'true' +BigQuery: + connector: 'bigquery' + type: 'append-only' + bigquery.local.path: 'xxx.json' + bigquery.project: 'xxx' + bigquery.dataset: 'test_bigquery_sink' + bigquery.table: 'table_bench' + force_append_only: 'true' \ No newline at end of file diff --git a/src/cmd/Cargo.toml b/src/cmd/Cargo.toml index de69cc3574509..10a007932dcf9 100644 --- a/src/cmd/Cargo.toml +++ b/src/cmd/Cargo.toml @@ -52,26 +52,6 @@ task_stats_alloc = { path = "../utils/task_stats_alloc" } [lib] test = false -[[bin]] -name = "frontend" -path = "src/bin/frontend_node.rs" -test = false - -[[bin]] -name = "meta-node" -path = "src/bin/meta_node.rs" -test = false - -[[bin]] -name = "compute-node" -path = "src/bin/compute_node.rs" -test = false - -[[bin]] -name = "compactor" -path = "src/bin/compactor.rs" -test = false - [[bin]] name = "risectl" path = "src/bin/ctl.rs" diff --git a/src/cmd/src/bin/meta_node.rs b/src/cmd/src/bin/meta_node.rs deleted file mode 100644 index bb474d014871e..0000000000000 --- a/src/cmd/src/bin/meta_node.rs +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#![cfg_attr(coverage, feature(coverage_attribute))] - -risingwave_cmd::main!(meta); diff --git a/src/cmd/src/lib.rs b/src/cmd/src/lib.rs index ce110c9effc17..a2f3457bb1266 100644 --- a/src/cmd/src/lib.rs +++ b/src/cmd/src/lib.rs @@ -63,17 +63,5 @@ pub fn compactor(opts: CompactorOpts) { pub fn ctl(opts: CtlOpts) { init_risingwave_logger(LoggerSettings::new("ctl").stderr(true)); - - // Note: Use a simple current thread runtime for ctl. - // When there's a heavy workload, multiple thread runtime seems to respond slowly. May need - // further investigation. - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap() - .block_on(risingwave_ctl::start(opts)) - .inspect_err(|e| { - eprintln!("{:#?}", e); - }) - .unwrap(); + main_okk(risingwave_ctl::start(opts)); } diff --git a/src/cmd_all/Cargo.toml b/src/cmd_all/Cargo.toml index e5aa9e3c68d04..813928ce931ea 100644 --- a/src/cmd_all/Cargo.toml +++ b/src/cmd_all/Cargo.toml @@ -10,6 +10,7 @@ repository = { workspace = true } [features] rw-static-link = ["workspace-config/rw-static-link"] rw-dynamic-link = ["workspace-config/rw-dynamic-link"] +embedded-python-udf = ["risingwave_expr/embedded-python-udf"] default = ["rw-static-link"] [package.metadata.cargo-machete] @@ -23,19 +24,21 @@ anyhow = "1" clap = { version = "4", features = ["cargo", "derive"] } console = "0.15" const-str = "0.5" +home = "0.5" prometheus = { version = "0.13" } risingwave_cmd = { workspace = true } risingwave_common = { workspace = true } risingwave_compactor = { workspace = true } risingwave_compute = { workspace = true } risingwave_ctl = { workspace = true } +risingwave_expr = { workspace = true } risingwave_expr_impl = { workspace = true } risingwave_frontend = { workspace = true } risingwave_meta_node = { workspace = true } risingwave_rt = { workspace = true } shell-words = "1.1.0" strum = "0.25" -strum_macros = "0.25" +strum_macros = "0.26" tempfile = "3" tikv-jemallocator = { workspace = true, features = [ "unprefixed_malloc_on_supported_platforms", @@ -57,6 +60,7 @@ workspace-hack = { path = "../workspace-hack" } expect-test = "1" [build-dependencies] +thiserror-ext = { workspace = true } vergen = { version = "8", default-features = false, features = [ "build", "git", diff --git a/src/cmd_all/build.rs b/src/cmd_all/build.rs index a4a7c27e65685..38d9f2d7107a6 100644 --- a/src/cmd_all/build.rs +++ b/src/cmd_all/build.rs @@ -12,11 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use thiserror_ext::AsReport; use vergen::EmitBuilder; fn main() { if let Err(e) = EmitBuilder::builder().git_sha(true).fail_on_error().emit() { // Leave the environment variable unset if error occurs. - println!("cargo:warning={}", e) + println!("cargo:warning={}", e.as_report()) } } diff --git a/src/cmd_all/src/bin/risingwave.rs b/src/cmd_all/src/bin/risingwave.rs index a1e3a1b5f7063..e9173abefe1df 100644 --- a/src/cmd_all/src/bin/risingwave.rs +++ b/src/cmd_all/src/bin/risingwave.rs @@ -17,9 +17,10 @@ use std::str::FromStr; use anyhow::Result; +use clap::error::ErrorKind; use clap::{command, ArgMatches, Args, Command, FromArgMatches}; use risingwave_cmd::{compactor, compute, ctl, frontend, meta}; -use risingwave_cmd_all::{PlaygroundOpts, StandaloneOpts}; +use risingwave_cmd_all::{PlaygroundOpts, SingleNodeOpts, StandaloneOpts}; use risingwave_common::git_sha; use risingwave_compactor::CompactorOpts; use risingwave_compute::ComputeNodeOpts; @@ -98,7 +99,14 @@ enum Component { Compactor, Ctl, Playground, + /// Used by cloud to bundle different components into a single node. + /// It exposes the low level configuration options of each node. Standalone, + /// Used by users to run a single node. + /// The low level configuration options are hidden. + /// We only expose high-level configuration options, + /// which map across multiple nodes. + SingleNode, } impl Component { @@ -117,6 +125,7 @@ impl Component { Self::Ctl => ctl(parse_opts(matches)), Self::Playground => playground(parse_opts(matches)), Self::Standalone => standalone(parse_opts(matches)), + Self::SingleNode => single_node(parse_opts(matches)), } } @@ -130,6 +139,7 @@ impl Component { Component::Ctl => vec!["risectl"], Component::Playground => vec!["play"], Component::Standalone => vec![], + Component::SingleNode => vec!["single-node", "single"], } } @@ -143,6 +153,7 @@ impl Component { Component::Ctl => CtlOpts::augment_args(cmd), Component::Playground => PlaygroundOpts::augment_args(cmd), Component::Standalone => StandaloneOpts::augment_args(cmd), + Component::SingleNode => SingleNodeOpts::augment_args(cmd), } } @@ -179,7 +190,23 @@ fn main() -> Result<()> { .subcommands(Component::commands()), ); - let matches = command.get_matches(); + let matches = match command.try_get_matches() { + Ok(m) => m, + Err(e) if e.kind() == ErrorKind::MissingSubcommand => { + // `$ ./risingwave` + // NOTE(kwannoel): This is a hack to make `risingwave` + // work as an alias of `risingwave single-process`. + // If invocation is not a multicall and there's no subcommand, + // we will try to invoke it as a single node. + let command = Component::SingleNode.augment_args(risingwave()); + let matches = command.get_matches(); + Component::SingleNode.start(&matches); + return Ok(()); + } + Err(e) => { + e.exit(); + } + }; let multicall = matches.subcommand().unwrap(); let argv_1 = multicall.1.subcommand(); @@ -207,3 +234,16 @@ fn standalone(opts: StandaloneOpts) { risingwave_rt::init_risingwave_logger(settings); risingwave_rt::main_okk(risingwave_cmd_all::standalone(opts)).unwrap(); } + +/// For single node, the internals are just a config mapping from its +/// high level options to standalone mode node-level options. +/// We will start a standalone instance, with all nodes in the same process. +fn single_node(opts: SingleNodeOpts) { + opts.create_store_directories().unwrap(); + let opts = risingwave_cmd_all::map_single_node_opts_to_standalone_opts(&opts); + let settings = risingwave_rt::LoggerSettings::from_opts(&opts) + .with_target("risingwave_storage", Level::WARN) + .with_thread_name(true); + risingwave_rt::init_risingwave_logger(settings); + risingwave_rt::main_okk(risingwave_cmd_all::standalone(opts)).unwrap(); +} diff --git a/src/cmd_all/src/lib.rs b/src/cmd_all/src/lib.rs index 94d4fd7ae3929..54ee3243bc662 100644 --- a/src/cmd_all/src/lib.rs +++ b/src/cmd_all/src/lib.rs @@ -18,7 +18,10 @@ mod common; pub mod playground; mod standalone; +pub mod single_node; + pub use playground::*; +pub use single_node::*; pub use standalone::*; risingwave_expr_impl::enable!(); diff --git a/src/cmd_all/src/playground.rs b/src/cmd_all/src/playground.rs index 70039264f6ef7..1b03048d5d6e0 100644 --- a/src/cmd_all/src/playground.rs +++ b/src/cmd_all/src/playground.rs @@ -137,7 +137,7 @@ fn get_services(profile: &str) -> (Vec, bool) { } #[derive(Debug, Clone, Parser)] -#[command(about = "The quick way to start a RisingWave cluster for playing around")] +#[command(about = "The quick way to start an in-memory RisingWave cluster for playing around")] pub struct PlaygroundOpts { /// The profile to use. #[clap(short, long, env = "PLAYGROUND_PROFILE", default_value = "playground")] diff --git a/src/cmd_all/src/single_node.rs b/src/cmd_all/src/single_node.rs new file mode 100644 index 0000000000000..042a0feee9863 --- /dev/null +++ b/src/cmd_all/src/single_node.rs @@ -0,0 +1,243 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::LazyLock; + +use anyhow::Result; +use clap::Parser; +use home::home_dir; +use risingwave_common::config::{AsyncStackTraceOption, MetaBackend}; +use risingwave_compactor::CompactorOpts; +use risingwave_compute::{default_parallelism, default_total_memory_bytes, ComputeNodeOpts}; +use risingwave_frontend::FrontendOpts; +use risingwave_meta_node::MetaNodeOpts; + +use crate::ParsedStandaloneOpts; + +pub static DEFAULT_STORE_DIRECTORY: LazyLock = LazyLock::new(|| { + let mut home_path = home_dir().unwrap(); + home_path.push(".risingwave"); + let home_path = home_path.to_str().unwrap(); + home_path.to_string() +}); + +pub static DEFAULT_SINGLE_NODE_SQLITE_PATH: LazyLock = + LazyLock::new(|| format!("{}/meta_store/single_node.db", &*DEFAULT_STORE_DIRECTORY)); + +pub static DEFAULT_SINGLE_NODE_SQL_ENDPOINT: LazyLock = + LazyLock::new(|| format!("sqlite://{}?mode=rwc", *DEFAULT_SINGLE_NODE_SQLITE_PATH)); + +pub static DEFAULT_SINGLE_NODE_STATE_STORE_PATH: LazyLock = + LazyLock::new(|| format!("{}/state_store", DEFAULT_STORE_DIRECTORY.clone())); + +pub static DEFAULT_SINGLE_NODE_STATE_STORE_URL: LazyLock = LazyLock::new(|| { + format!( + "hummock+fs://{}", + DEFAULT_SINGLE_NODE_STATE_STORE_PATH.clone() + ) +}); + +#[derive(Eq, PartialOrd, PartialEq, Debug, Clone, Parser)] +#[command( + version, + about = "[default] The Single Node mode. Start all services in one process, with process-level options. This will be executed if no subcommand is specified" +)] +/// Here we define our own defaults for the single node mode. +pub struct SingleNodeOpts { + /// The address prometheus polls metrics from. + #[clap(long, env = "RW_SINGLE_NODE_PROMETHEUS_LISTENER_ADDR")] + prometheus_listener_addr: Option, + + /// The path to the cluster configuration file. + #[clap(long, env = "RW_SINGLE_NODE_CONFIG_PATH")] + config_path: Option, + + /// The store directory used by meta store and object store. + #[clap(long, env = "RW_SINGLE_NODE_STORE_DIRECTORY")] + pub store_directory: Option, + + /// The address of the meta node. + #[clap(long, env = "RW_SINGLE_NODE_META_ADDR")] + meta_addr: Option, + + /// The address of the compute node + #[clap(long, env = "RW_SINGLE_NODE_COMPUTE_ADDR")] + compute_addr: Option, + + /// The address of the frontend node + #[clap(long, env = "RW_SINGLE_NODE_FRONTEND_ADDR")] + frontend_addr: Option, + + /// The address of the compactor node + #[clap(long, env = "RW_SINGLE_NODE_COMPACTOR_ADDR")] + compactor_addr: Option, +} + +pub fn make_single_node_sql_endpoint(store_directory: &String) -> String { + format!( + "sqlite://{}/meta_store/single_node.db?mode=rwc", + store_directory + ) +} + +pub fn make_single_node_state_store_url(store_directory: &String) -> String { + format!("hummock+fs://{}/state_store", store_directory) +} + +pub fn map_single_node_opts_to_standalone_opts(opts: &SingleNodeOpts) -> ParsedStandaloneOpts { + let mut meta_opts = SingleNodeOpts::default_meta_opts(); + let mut compute_opts = SingleNodeOpts::default_compute_opts(); + let mut frontend_opts = SingleNodeOpts::default_frontend_opts(); + let mut compactor_opts = SingleNodeOpts::default_compactor_opts(); + if let Some(prometheus_listener_addr) = &opts.prometheus_listener_addr { + meta_opts.prometheus_listener_addr = Some(prometheus_listener_addr.clone()); + compute_opts.prometheus_listener_addr = prometheus_listener_addr.clone(); + frontend_opts.prometheus_listener_addr = prometheus_listener_addr.clone(); + compactor_opts.prometheus_listener_addr = prometheus_listener_addr.clone(); + } + if let Some(config_path) = &opts.config_path { + meta_opts.config_path = config_path.clone(); + compute_opts.config_path = config_path.clone(); + frontend_opts.config_path = config_path.clone(); + compactor_opts.config_path = config_path.clone(); + } + if let Some(store_directory) = &opts.store_directory { + let state_store_url = make_single_node_state_store_url(store_directory); + let meta_store_endpoint = make_single_node_sql_endpoint(store_directory); + meta_opts.state_store = Some(state_store_url); + meta_opts.sql_endpoint = Some(meta_store_endpoint); + } + if let Some(meta_addr) = &opts.meta_addr { + meta_opts.listen_addr = meta_addr.clone(); + meta_opts.advertise_addr = meta_addr.clone(); + + compute_opts.meta_address = meta_addr.parse().unwrap(); + frontend_opts.meta_addr = meta_addr.parse().unwrap(); + compactor_opts.meta_address = meta_addr.parse().unwrap(); + } + if let Some(compute_addr) = &opts.compute_addr { + compute_opts.listen_addr = compute_addr.clone(); + } + if let Some(frontend_addr) = &opts.frontend_addr { + frontend_opts.listen_addr = frontend_addr.clone(); + } + if let Some(compactor_addr) = &opts.compactor_addr { + compactor_opts.listen_addr = compactor_addr.clone(); + } + ParsedStandaloneOpts { + meta_opts: Some(meta_opts), + compute_opts: Some(compute_opts), + frontend_opts: Some(frontend_opts), + compactor_opts: Some(compactor_opts), + } +} + +// Defaults +impl SingleNodeOpts { + fn default_frontend_opts() -> FrontendOpts { + FrontendOpts { + listen_addr: "0.0.0.0:4566".to_string(), + advertise_addr: Some("0.0.0.0:4566".to_string()), + port: None, + meta_addr: "http://0.0.0.0:5690".parse().unwrap(), + prometheus_listener_addr: "0.0.0.0:1250".to_string(), + health_check_listener_addr: "0.0.0.0:6786".to_string(), + config_path: "".to_string(), + metrics_level: None, + enable_barrier_read: None, + } + } + + fn default_meta_opts() -> MetaNodeOpts { + MetaNodeOpts { + vpc_id: None, + security_group_id: None, + listen_addr: "0.0.0.0:5690".to_string(), + advertise_addr: "0.0.0.0:5690".to_string(), + dashboard_host: Some("0.0.0.0:5691".to_string()), + prometheus_listener_addr: Some("0.0.0.0:1250".to_string()), + etcd_endpoints: Default::default(), + etcd_auth: false, + etcd_username: Default::default(), + etcd_password: Default::default(), + sql_endpoint: Some(DEFAULT_SINGLE_NODE_SQL_ENDPOINT.clone()), + dashboard_ui_path: None, + prometheus_endpoint: None, + prometheus_selector: None, + connector_rpc_endpoint: None, + privatelink_endpoint_default_tags: None, + config_path: "".to_string(), + backend: Some(MetaBackend::Sql), + barrier_interval_ms: None, + sstable_size_mb: None, + block_size_kb: None, + bloom_false_positive: None, + state_store: Some(DEFAULT_SINGLE_NODE_STATE_STORE_URL.clone()), + data_directory: Some("hummock_001".to_string()), + do_not_config_object_storage_lifecycle: None, + backup_storage_url: None, + backup_storage_directory: None, + heap_profiling_dir: None, + } + } + + pub fn default_compute_opts() -> ComputeNodeOpts { + ComputeNodeOpts { + listen_addr: "0.0.0.0:5688".to_string(), + advertise_addr: Some("0.0.0.0:5688".to_string()), + prometheus_listener_addr: "0.0.0.0:1250".to_string(), + meta_address: "http://0.0.0.0:5690".parse().unwrap(), + connector_rpc_endpoint: None, + connector_rpc_sink_payload_format: None, + config_path: "".to_string(), + total_memory_bytes: default_total_memory_bytes(), + parallelism: default_parallelism(), + role: Default::default(), + metrics_level: None, + data_file_cache_dir: None, + meta_file_cache_dir: None, + async_stack_trace: Some(AsyncStackTraceOption::ReleaseVerbose), + heap_profiling_dir: None, + } + } + + fn default_compactor_opts() -> CompactorOpts { + CompactorOpts { + listen_addr: "0.0.0.0:6660".to_string(), + advertise_addr: Some("0.0.0.0:6660".to_string()), + port: None, + prometheus_listener_addr: "0.0.0.0:1250".to_string(), + meta_address: "http://0.0.0.0:5690".parse().unwrap(), + compaction_worker_threads_number: None, + config_path: "".to_string(), + metrics_level: None, + async_stack_trace: None, + heap_profiling_dir: None, + compactor_mode: None, + proxy_rpc_endpoint: "".to_string(), + } + } +} + +impl SingleNodeOpts { + pub fn create_store_directories(&self) -> Result<()> { + let store_directory = self + .store_directory + .as_ref() + .unwrap_or_else(|| &*DEFAULT_STORE_DIRECTORY); + std::fs::create_dir_all(format!("{}/meta_store", store_directory))?; + std::fs::create_dir_all(format!("{}/state_store", store_directory))?; + Ok(()) + } +} diff --git a/src/cmd_all/src/standalone.rs b/src/cmd_all/src/standalone.rs index a51fb03120313..33e61e5b41f2b 100644 --- a/src/cmd_all/src/standalone.rs +++ b/src/cmd_all/src/standalone.rs @@ -25,6 +25,11 @@ use tokio::signal; use crate::common::osstrs; #[derive(Eq, PartialOrd, PartialEq, Debug, Clone, Parser)] +#[command( + version, + about = "The Standalone mode allows users to start multiple services in one process, it exposes node-level options for each service", + hide = true +)] pub struct StandaloneOpts { /// Compute node options /// If missing, compute node won't start @@ -141,7 +146,7 @@ pub fn parse_standalone_opt_args(opts: &StandaloneOpts) -> ParsedStandaloneOpts compactor_opts.prometheus_listener_addr = prometheus_listener_addr.clone(); } if let Some(meta_opts) = meta_opts.as_mut() { - meta_opts.prometheus_host = Some(prometheus_listener_addr.clone()); + meta_opts.prometheus_listener_addr = Some(prometheus_listener_addr.clone()); } } @@ -161,6 +166,9 @@ pub fn parse_standalone_opt_args(opts: &StandaloneOpts) -> ParsedStandaloneOpts } } +/// For `standalone` mode, we can configure and start multiple services in one process. +/// `standalone` mode is meant to be used by our cloud service and docker, +/// where we can configure and start multiple services in one process. pub async fn standalone( ParsedStandaloneOpts { meta_opts, @@ -246,6 +254,7 @@ mod test { // Test parsing into node-level opts. let actual = parse_standalone_opt_args(&opts); + check( actual, expect![[r#" @@ -257,7 +266,7 @@ mod test { listen_addr: "127.0.0.1:8001", advertise_addr: "127.0.0.1:9999", dashboard_host: None, - prometheus_host: Some( + prometheus_listener_addr: Some( "127.0.0.1:1234", ), etcd_endpoints: "", @@ -300,7 +309,6 @@ mod test { connector_rpc_sink_payload_format: None, config_path: "src/config/test.toml", total_memory_bytes: 34359738368, - mem_table_spill_threshold: 4194304, parallelism: 10, role: Both, metrics_level: None, diff --git a/src/common/Cargo.toml b/src/common/Cargo.toml index cb859665fb6d6..61bca47e1018f 100644 --- a/src/common/Cargo.toml +++ b/src/common/Cargo.toml @@ -44,7 +44,7 @@ either = "1" enum-as-inner = "0.6" enumflags2 = { version = "0.7.8" } ethnum = { version = "1", features = ["serde"] } -fixedbitset = { version = "0.4", features = ["std"] } +fixedbitset = { version = "0.5", features = ["std"] } fs-err = "2" futures = { version = "0.3", default-features = false, features = ["alloc"] } governor = { version = "0.6", default-features = false, features = ["std"] } @@ -56,14 +56,15 @@ hytra = { workspace = true } itertools = "0.12" itoa = "1.0" jsonbb = "0.1.2" -lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "cb2d7c7" } +lru = { workspace = true } memcomparable = { version = "0.2", features = ["decimal"] } num-integer = "0.1" num-traits = "0.2" number_prefix = "0.4.0" -opentelemetry = { version = "0.20", default-features = false } +opentelemetry = { workspace = true } +opentelemetry_sdk = { workspace = true } parking_lot = "0.12" -parse-display = "0.8" +parse-display = "0.9" paste = "1" pin-project-lite = "0.2" postgres-types = { version = "0.2.6", features = [ @@ -82,6 +83,7 @@ risingwave_common_proc_macro = { path = "./proc_macro" } risingwave_error = { workspace = true } risingwave_pb = { workspace = true } rust_decimal = { version = "1", features = ["db-postgres", "maths"] } +rw_futures_util = { workspace = true } ryu = "1.0" serde = { version = "1", features = ["derive"] } serde_bytes = "0.11" @@ -92,7 +94,7 @@ smallbitset = "0.7.1" speedate = "0.13.0" static_assertions = "1" strum = "0.25" -strum_macros = "0.25" +strum_macros = "0.26" sysinfo = { version = "0.30", default-features = false } thiserror = "1" thiserror-ext = { workspace = true } @@ -109,7 +111,7 @@ toml = "0.8" tonic = { workspace = true } tracing = "0.1" tracing-futures = { version = "0.2", features = ["futures-03"] } -tracing-opentelemetry = "0.21" +tracing-opentelemetry = { workspace = true } tracing-subscriber = "0.3.17" twox-hash = "1" url = "2" diff --git a/src/common/benches/bench_hash_key_encoding.rs b/src/common/benches/bench_hash_key_encoding.rs index 185b5ebcc8c47..c1529a212ac9e 100644 --- a/src/common/benches/bench_hash_key_encoding.rs +++ b/src/common/benches/bench_hash_key_encoding.rs @@ -81,7 +81,7 @@ impl HashKeyBenchCase { // please reference the `bench_vec_deser` and `bench_deser` method for benchmarking partial // `col_idxes` let col_idxes = (0..input_chunk.columns().len()).collect_vec(); - let keys = HashKey::build(&col_idxes, &input_chunk).unwrap(); + let keys = HashKey::build_many(&col_idxes, &input_chunk); Self { id, input_chunk, @@ -94,7 +94,7 @@ impl HashKeyBenchCase { pub fn bench_vec_ser(&self, c: &mut Criterion) { let vectorize_serialize_id = "vec ser ".to_string() + &self.id; c.bench_function(&vectorize_serialize_id, |b| { - b.iter(|| K::build(&self.col_idxes, &self.input_chunk).unwrap()) + b.iter(|| K::build_many(&self.col_idxes, &self.input_chunk)) }); } diff --git a/src/common/fields-derive/src/gen/test_empty_pk.rs b/src/common/fields-derive/src/gen/test_empty_pk.rs new file mode 100644 index 0000000000000..ffb5ff268bed1 --- /dev/null +++ b/src/common/fields-derive/src/gen/test_empty_pk.rs @@ -0,0 +1,29 @@ +impl ::risingwave_common::types::Fields for Data { + const PRIMARY_KEY: Option<&'static [usize]> = Some(&[]); + fn fields() -> Vec<(&'static str, ::risingwave_common::types::DataType)> { + vec![ + ("v1", < i16 as ::risingwave_common::types::WithDataType > + ::default_data_type()), ("v2", < String as + ::risingwave_common::types::WithDataType > ::default_data_type()) + ] + } + fn into_owned_row(self) -> ::risingwave_common::row::OwnedRow { + ::risingwave_common::row::OwnedRow::new( + vec![ + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v1), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v2) + ], + ) + } +} +impl From for ::risingwave_common::types::ScalarImpl { + fn from(v: Data) -> Self { + ::risingwave_common::types::StructValue::new( + vec![ + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v1), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v2) + ], + ) + .into() + } +} diff --git a/src/common/fields-derive/src/gen/test_no_pk.rs b/src/common/fields-derive/src/gen/test_no_pk.rs new file mode 100644 index 0000000000000..9e1b3e7892969 --- /dev/null +++ b/src/common/fields-derive/src/gen/test_no_pk.rs @@ -0,0 +1,29 @@ +impl ::risingwave_common::types::Fields for Data { + const PRIMARY_KEY: Option<&'static [usize]> = None; + fn fields() -> Vec<(&'static str, ::risingwave_common::types::DataType)> { + vec![ + ("v1", < i16 as ::risingwave_common::types::WithDataType > + ::default_data_type()), ("v2", < String as + ::risingwave_common::types::WithDataType > ::default_data_type()) + ] + } + fn into_owned_row(self) -> ::risingwave_common::row::OwnedRow { + ::risingwave_common::row::OwnedRow::new( + vec![ + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v1), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v2) + ], + ) + } +} +impl From for ::risingwave_common::types::ScalarImpl { + fn from(v: Data) -> Self { + ::risingwave_common::types::StructValue::new( + vec![ + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v1), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v2) + ], + ) + .into() + } +} diff --git a/src/common/fields-derive/src/gen/test_output.rs b/src/common/fields-derive/src/gen/test_output.rs index d8e7274b4c2ee..a804a379bfd4a 100644 --- a/src/common/fields-derive/src/gen/test_output.rs +++ b/src/common/fields-derive/src/gen/test_output.rs @@ -1,4 +1,5 @@ impl ::risingwave_common::types::Fields for Data { + const PRIMARY_KEY: Option<&'static [usize]> = Some(&[1usize, 0usize]); fn fields() -> Vec<(&'static str, ::risingwave_common::types::DataType)> { vec![ ("v1", < i16 as ::risingwave_common::types::WithDataType > @@ -6,7 +7,33 @@ impl ::risingwave_common::types::Fields for Data { ::risingwave_common::types::WithDataType > ::default_data_type()), ("v3", < bool as ::risingwave_common::types::WithDataType > ::default_data_type()), ("v4", < Serial as ::risingwave_common::types::WithDataType > - ::default_data_type()) + ::default_data_type()), ("type", < i32 as + ::risingwave_common::types::WithDataType > ::default_data_type()) ] } + fn into_owned_row(self) -> ::risingwave_common::row::OwnedRow { + ::risingwave_common::row::OwnedRow::new( + vec![ + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v1), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v2), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v3), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.v4), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.r#type) + ], + ) + } +} +impl From for ::risingwave_common::types::ScalarImpl { + fn from(v: Data) -> Self { + ::risingwave_common::types::StructValue::new( + vec![ + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v1), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v2), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v3), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.v4), + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.r#type) + ], + ) + .into() + } } diff --git a/src/common/fields-derive/src/lib.rs b/src/common/fields-derive/src/lib.rs index 9b30c4ee72419..dae648d1dc343 100644 --- a/src/common/fields-derive/src/lib.rs +++ b/src/common/fields-derive/src/lib.rs @@ -14,9 +14,9 @@ use proc_macro2::TokenStream; use quote::quote; -use syn::{Data, DeriveInput, Field, Result}; +use syn::{Data, DeriveInput, Result}; -#[proc_macro_derive(Fields)] +#[proc_macro_derive(Fields, attributes(primary_key, fields))] pub fn fields(tokens: proc_macro::TokenStream) -> proc_macro::TokenStream { inner(tokens.into()).into() } @@ -31,51 +31,165 @@ fn inner(tokens: TokenStream) -> TokenStream { fn gen(tokens: TokenStream) -> Result { let input: DeriveInput = syn::parse2(tokens)?; - let DeriveInput { - attrs: _attrs, - vis: _vis, - ident, - generics, - data, - } = input; - if !generics.params.is_empty() { + let ident = &input.ident; + if !input.generics.params.is_empty() { return Err(syn::Error::new_spanned( - generics, + input.generics, "generics are not supported", )); } - let Data::Struct(r#struct) = data else { - return Err(syn::Error::new_spanned(ident, "only structs are supported")); + let Data::Struct(struct_) = &input.data else { + return Err(syn::Error::new_spanned( + input.ident, + "only structs are supported", + )); }; - let fields_rs = r#struct.fields; - let fields_rw: Vec = fields_rs - .into_iter() - .map(|field_rs| { - let Field { - // We can support #[field(ignore)] or other useful attributes here. - attrs: _attrs, - ident: name, - ty, - .. - } = field_rs; - let name = name.map_or("".to_string(), |name| name.to_string()); + let style = get_style(&input); + if let Some(style) = &style { + if !["Title Case", "TITLE CASE", "snake_case"].contains(&style.value().as_str()) { + return Err(syn::Error::new_spanned( + style, + "only `Title Case`, `TITLE CASE`, and `snake_case` are supported", + )); + } + } + + let fields_rw: Vec = struct_ + .fields + .iter() + .map(|field| { + let mut name = field.ident.as_ref().expect("field no name").to_string(); + // strip leading `r#` + if name.starts_with("r#") { + name = name[2..].to_string(); + } + // cast style + match style.as_ref().map_or(String::new(), |f| f.value()).as_str() { + "Title Case" => name = to_title_case(&name), + "TITLE CASE" => name = to_title_case(&name).to_uppercase(), + _ => {} + } + let ty = &field.ty; quote! { (#name, <#ty as ::risingwave_common::types::WithDataType>::default_data_type()) } }) .collect(); + let names = struct_ + .fields + .iter() + .map(|field| field.ident.as_ref().expect("field no name")) + .collect::>(); + let primary_key = get_primary_key(&input).map_or_else( + || quote! { None }, + |indices| { + quote! { Some(&[#(#indices),*]) } + }, + ); Ok(quote! { impl ::risingwave_common::types::Fields for #ident { + const PRIMARY_KEY: Option<&'static [usize]> = #primary_key; + fn fields() -> Vec<(&'static str, ::risingwave_common::types::DataType)> { vec![#(#fields_rw),*] } + fn into_owned_row(self) -> ::risingwave_common::row::OwnedRow { + ::risingwave_common::row::OwnedRow::new(vec![#( + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(self.#names) + ),*]) + } + } + impl From<#ident> for ::risingwave_common::types::ScalarImpl { + fn from(v: #ident) -> Self { + ::risingwave_common::types::StructValue::new(vec![#( + ::risingwave_common::types::ToOwnedDatum::to_owned_datum(v.#names) + ),*]).into() + } } }) } +/// Get primary key indices from `#[primary_key]` attribute. +fn get_primary_key(input: &syn::DeriveInput) -> Option> { + let syn::Data::Struct(struct_) = &input.data else { + return None; + }; + // find `#[primary_key(k1, k2, ...)]` on struct + let composite = input.attrs.iter().find_map(|attr| match &attr.meta { + syn::Meta::List(list) if list.path.is_ident("primary_key") => Some(&list.tokens), + _ => None, + }); + if let Some(keys) = composite { + let index = |name: &str| { + struct_ + .fields + .iter() + .position(|f| f.ident.as_ref().map_or(false, |i| i == name)) + .expect("primary key not found") + }; + return Some( + keys.to_string() + .split(',') + .map(|s| s.trim()) + .filter(|s| !s.is_empty()) + .map(index) + .collect(), + ); + } + // find `#[primary_key]` on fields + for (i, field) in struct_.fields.iter().enumerate() { + for attr in &field.attrs { + if matches!(&attr.meta, syn::Meta::Path(path) if path.is_ident("primary_key")) { + return Some(vec![i]); + } + } + } + None +} + +/// Get name style from `#[fields(style = "xxx")]` attribute. +fn get_style(input: &syn::DeriveInput) -> Option { + let style = input.attrs.iter().find_map(|attr| match &attr.meta { + syn::Meta::List(list) if list.path.is_ident("fields") => { + let name_value: syn::MetaNameValue = syn::parse2(list.tokens.clone()).ok()?; + if name_value.path.is_ident("style") { + Some(name_value.value) + } else { + None + } + } + _ => None, + })?; + match style { + syn::Expr::Lit(lit) => match lit.lit { + syn::Lit::Str(s) => Some(s), + _ => None, + }, + _ => None, + } +} + +/// Convert `snake_case` to `Title Case`. +fn to_title_case(s: &str) -> String { + let mut title = String::new(); + let mut next_upper = true; + for c in s.chars() { + if c == '_' { + title.push(' '); + next_upper = true; + } else if next_upper { + title.push(c.to_uppercase().next().unwrap()); + next_upper = false; + } else { + title.push(c); + } + } + title +} + #[cfg(test)] mod tests { use indoc::indoc; @@ -87,26 +201,59 @@ mod tests { prettyplease::unparse(&output) } + fn do_test(code: &str, expected_path: &str) { + let input: TokenStream = str::parse(code).unwrap(); + + let output = super::gen(input).unwrap(); + + let output = pretty_print(output); + + let expected = expect_test::expect_file![expected_path]; + + expected.assert_eq(&output); + } + #[test] fn test_gen() { let code = indoc! {r#" #[derive(Fields)] + #[primary_key(v2, v1)] struct Data { v1: i16, v2: std::primitive::i32, v3: bool, v4: Serial, + r#type: i32, } "#}; - let input: TokenStream = str::parse(code).unwrap(); + do_test(code, "gen/test_output.rs"); + } - let output = super::gen(input).unwrap(); + #[test] + fn test_no_pk() { + let code = indoc! {r#" + #[derive(Fields)] + struct Data { + v1: i16, + v2: String, + } + "#}; - let output = pretty_print(output); + do_test(code, "gen/test_no_pk.rs"); + } - let expected = expect_test::expect_file!["gen/test_output.rs"]; + #[test] + fn test_empty_pk() { + let code = indoc! {r#" + #[derive(Fields)] + #[primary_key()] + struct Data { + v1: i16, + v2: String, + } + "#}; - expected.assert_eq(&output); + do_test(code, "gen/test_empty_pk.rs"); } } diff --git a/src/common/proc_macro/Cargo.toml b/src/common/proc_macro/Cargo.toml index 0f86bff6a19d2..bbfe10baf3b07 100644 --- a/src/common/proc_macro/Cargo.toml +++ b/src/common/proc_macro/Cargo.toml @@ -22,6 +22,7 @@ quote = "1" proc-macro2 = { version = "1", default-features = false } syn = "1" bae = "0.1.7" +itertools = "0.12" [lints] workspace = true diff --git a/src/common/proc_macro/src/config_doc.rs b/src/common/proc_macro/src/config_doc.rs new file mode 100644 index 0000000000000..e63145087f157 --- /dev/null +++ b/src/common/proc_macro/src/config_doc.rs @@ -0,0 +1,157 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use itertools::Itertools; +use quote::quote; +use syn::{Attribute, Data, DataStruct, DeriveInput, Field, Fields}; + +pub fn generate_config_doc_fn(input: DeriveInput) -> proc_macro2::TokenStream { + let mut doc = StructFieldDocs::new(); + + let struct_name = input.ident; + match input.data { + Data::Struct(ref data) => doc.extract_field_docs(data), + _ => panic!("This macro only supports structs"), + }; + + let vec_fields = doc.token_vec_fields(); + let call_nested_fields = doc.token_call_nested_fields(); + quote! { + impl #struct_name { + pub fn config_docs(name: String, docs: &mut std::collections::BTreeMap>) { + docs.insert(name.clone(), #vec_fields); + #call_nested_fields; + } + } + } +} + +fn extract_comment(attrs: &Vec) -> String { + attrs + .iter() + .filter_map(|attr| { + if let Ok(meta) = attr.parse_meta() { + if meta.path().is_ident("doc") { + if let syn::Meta::NameValue(syn::MetaNameValue { + lit: syn::Lit::Str(comment), + .. + }) = meta + { + return Some(comment.value()); + } + } + } + None + }) + .join(" ") +} + +fn is_nested_config_field(field: &Field) -> bool { + field.attrs.iter().any(|attr| { + if let Some(attr_name) = attr.path.get_ident() { + attr_name == "config_doc" && attr.tokens.to_string() == "(nested)" + } else { + false + } + }) +} + +fn is_omitted_config_field(field: &Field) -> bool { + field.attrs.iter().any(|attr| { + if let Some(attr_name) = attr.path.get_ident() { + attr_name == "config_doc" && attr.tokens.to_string() == "(omitted)" + } else { + false + } + }) +} + +fn field_name(f: &Field) -> String { + f.ident + .as_ref() + .expect("field name should not be empty") + .to_string() +} + +struct StructFieldDocs { + // Fields that require recursively retrieving their field docs. + nested_fields: Vec<(String, syn::Type)>, + + fields: Vec<(String, String)>, +} + +impl StructFieldDocs { + fn new() -> Self { + Self { + nested_fields: vec![], + fields: vec![], + } + } + + fn extract_field_docs(&mut self, data: &DataStruct) { + match &data.fields { + Fields::Named(fields) => { + self.fields = fields + .named + .iter() + .filter_map(|field| { + if is_omitted_config_field(field) { + return None; + } + if is_nested_config_field(field) { + self.nested_fields + .push((field_name(field), field.ty.clone())); + return None; + } + let field_name = field.ident.as_ref()?.to_string(); + let rustdoc = extract_comment(&field.attrs); + Some((field_name, rustdoc)) + }) + .collect_vec(); + } + _ => unreachable!("field should be named"), + } + } + + fn token_vec_fields(&self) -> proc_macro2::TokenStream { + let token_fields: Vec = self + .fields + .iter() + .map(|(name, doc)| { + quote! { (#name.to_string(), #doc.to_string()) } + }) + .collect(); + + quote! { + vec![#(#token_fields),*] + } + } + + fn token_call_nested_fields(&self) -> proc_macro2::TokenStream { + let tokens: Vec = self + .nested_fields + .iter() + .map(|(ident, ty)| { + quote! { + if name.is_empty() { + #ty::config_docs(#ident.to_string(), docs); + } else { + #ty::config_docs(format!("{}.{}", name, #ident), docs); + } + } + }) + .collect(); + quote! { #(#tokens)* } + } +} diff --git a/src/common/proc_macro/src/lib.rs b/src/common/proc_macro/src/lib.rs index 77807f8291ff2..cb52bf9786fef 100644 --- a/src/common/proc_macro/src/lib.rs +++ b/src/common/proc_macro/src/lib.rs @@ -23,6 +23,7 @@ use quote::quote; use syn::parse_macro_input; mod config; +mod config_doc; mod estimate_size; mod session_config; @@ -263,3 +264,44 @@ pub fn session_config(input: TokenStream) -> TokenStream { let input = parse_macro_input!(input); session_config::derive_config(input).into() } + +/// This proc macro recursively extracts rustdoc comments from the fields in a struct and generates a method +/// that produces docs for each field. +/// Unlike rustdoc, this tool focuses solely on extracting rustdoc for struct fields, without methods. +/// +/// Example: +/// +/// ```ignore +/// #[derive(ConfigDoc)] +/// pub struct Foo { +/// /// Description for `a`. +/// a: i32, +/// +/// #[config_doc(nested)] +/// b: Bar, +/// +/// #[config_doc(omitted)] +/// dummy: (), +/// } +/// ``` +/// +/// The `#[config_doc(nested)]` attribute indicates that the field is a nested config that will be documented in a separate section. +/// Fields marked with `#[config_doc(omitted)]` will simply be omitted from the doc. +/// +/// Here is the method generated by this macro: +/// +/// ```ignore +/// impl Foo { +/// pub fn config_docs(name: String, docs: &mut std::collections::BTreeMap>) +/// } +/// ``` +/// +/// In `test_example_up_to_date`, we further process the output of this method to generate a markdown in src/config/docs.md. +#[proc_macro_derive(ConfigDoc, attributes(config_doc))] +pub fn config_doc(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input); + + let gen = config_doc::generate_config_doc_fn(input); + + gen.into() +} diff --git a/src/common/src/array/arrow/arrow_iceberg.rs b/src/common/src/array/arrow/arrow_iceberg.rs new file mode 100644 index 0000000000000..73476a8d291a7 --- /dev/null +++ b/src/common/src/array/arrow/arrow_iceberg.rs @@ -0,0 +1,73 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use arrow_array::{ArrayRef, StructArray}; +use arrow_schema::DataType; +use itertools::Itertools; + +use crate::array::{ArrayError, DataChunk}; +use crate::util::iter_util::ZipEqFast; + +/// Converts RisingWave array to Arrow array with the schema. +/// The behavior is specified for iceberg: +/// For different struct type, try to use fields in schema to cast. +pub fn to_iceberg_record_batch_with_schema( + schema: arrow_schema::SchemaRef, + chunk: &DataChunk, +) -> Result { + if !chunk.is_compacted() { + let c = chunk.clone(); + return to_iceberg_record_batch_with_schema(schema, &c.compact()); + } + let columns: Vec<_> = chunk + .columns() + .iter() + .zip_eq_fast(schema.fields().iter()) + .map(|(column, field)| { + let column: arrow_array::ArrayRef = column.as_ref().try_into()?; + if column.data_type() == field.data_type() { + Ok(column) + } else if let DataType::Struct(actual) = column.data_type() + && let DataType::Struct(expect) = field.data_type() + { + // Special case for iceberg + if actual.len() != expect.len() { + return Err(ArrayError::to_arrow(format!( + "Struct field count mismatch, expect {}, actual {}", + expect.len(), + actual.len() + ))); + } + let column = column + .as_any() + .downcast_ref::() + .unwrap() + .clone(); + let (_, struct_columns, nullable) = column.into_parts(); + Ok(Arc::new( + StructArray::try_new(expect.clone(), struct_columns, nullable) + .map_err(ArrayError::from_arrow)?, + ) as ArrayRef) + } else { + arrow_cast::cast(&column, field.data_type()).map_err(ArrayError::from_arrow) + } + }) + .try_collect::<_, _, ArrayError>()?; + + let opts = arrow_array::RecordBatchOptions::default().with_row_count(Some(chunk.capacity())); + arrow_array::RecordBatch::try_new_with_options(schema, columns, &opts) + .map_err(ArrayError::to_arrow) +} diff --git a/src/common/src/array/arrow/arrow_impl.rs b/src/common/src/array/arrow/arrow_impl.rs index 4eae49ac6ac00..39991bacc48d8 100644 --- a/src/common/src/array/arrow/arrow_impl.rs +++ b/src/common/src/array/arrow/arrow_impl.rs @@ -133,6 +133,7 @@ macro_rules! converts_generic { fn try_from(array: &ArrayImpl) -> Result { match array { $($ArrayImplPattern(a) => Ok(Arc::new(<$ArrowType>::try_from(a)?)),)* + ArrayImpl::Timestamptz(a) => Ok(Arc::new(arrow_array::TimestampMicrosecondArray::try_from(a)?. with_timezone_utc())), _ => todo!("unsupported array"), } } @@ -152,6 +153,21 @@ macro_rules! converts_generic { .unwrap() .try_into()?, )),)* + Timestamp(Microsecond, Some(_)) => Ok(ArrayImpl::Timestamptz( + array + .as_any() + .downcast_ref::() + .unwrap() + .try_into()?, + )), + // This arrow decimal type is used by iceberg source to read iceberg decimal into RW decimal. + Decimal128(_, _) => Ok(ArrayImpl::Decimal( + array + .as_any() + .downcast_ref::() + .unwrap() + .try_into()?, + )), t => Err(ArrayError::from_arrow(format!("unsupported data type: {t:?}"))), } } @@ -173,7 +189,6 @@ converts_generic! { { arrow_array::Decimal256Array, Decimal256(_, _), ArrayImpl::Int256 }, { arrow_array::Date32Array, Date32, ArrayImpl::Date }, { arrow_array::TimestampMicrosecondArray, Timestamp(Microsecond, None), ArrayImpl::Timestamp }, - { arrow_array::TimestampMicrosecondArray, Timestamp(Microsecond, Some(_)), ArrayImpl::Timestamptz }, { arrow_array::Time64MicrosecondArray, Time64(Microsecond), ArrayImpl::Time }, { arrow_array::IntervalMonthDayNanoArray, Interval(MonthDayNano), ArrayImpl::Interval }, { arrow_array::StructArray, Struct(_), ArrayImpl::Struct }, @@ -207,6 +222,7 @@ impl From<&arrow_schema::DataType> for DataType { LargeUtf8 => Self::Jsonb, Struct(fields) => Self::Struct(fields.into()), List(field) => Self::List(Box::new(field.data_type().into())), + Decimal128(_, _) => Self::Decimal, _ => todo!("Unsupported arrow data type: {value:?}"), } } @@ -499,6 +515,30 @@ impl From<&DecimalArray> for arrow_array::LargeBinaryArray { } } +// This arrow decimal type is used by iceberg source to read iceberg decimal into RW decimal. +impl TryFrom<&arrow_array::Decimal128Array> for DecimalArray { + type Error = ArrayError; + + fn try_from(array: &arrow_array::Decimal128Array) -> Result { + if array.scale() < 0 { + bail!("support negative scale for arrow decimal") + } + let from_arrow = |value| { + const NAN: i128 = i128::MIN + 1; + match value { + NAN => Decimal::NaN, + i128::MAX => Decimal::PositiveInf, + i128::MIN => Decimal::NegativeInf, + _ => Decimal::Normalized(rust_decimal::Decimal::from_i128_with_scale( + value, + array.scale() as u32, + )), + } + }; + Ok(array.iter().map(|o| o.map(from_arrow)).collect()) + } +} + impl TryFrom<&arrow_array::LargeBinaryArray> for DecimalArray { type Error = ArrayError; @@ -880,7 +920,7 @@ mod tests { #[test] fn int256() { - let values = vec![ + let values = [ None, Some(Int256::from(1)), Some(Int256::from(i64::MAX)), diff --git a/src/common/src/array/arrow/mod.rs b/src/common/src/array/arrow/mod.rs index 2e0db203a5ee1..b3fed6bafaa27 100644 --- a/src/common/src/array/arrow/mod.rs +++ b/src/common/src/array/arrow/mod.rs @@ -14,6 +14,8 @@ mod arrow_default; mod arrow_deltalake; +mod arrow_iceberg; pub use arrow_default::to_record_batch_with_schema; pub use arrow_deltalake::to_deltalake_record_batch_with_schema; +pub use arrow_iceberg::to_iceberg_record_batch_with_schema; diff --git a/src/common/src/array/mod.rs b/src/common/src/array/mod.rs index ef2caa8daa26a..f1012782bf9a6 100644 --- a/src/common/src/array/mod.rs +++ b/src/common/src/array/mod.rs @@ -15,7 +15,10 @@ //! `Array` defines all in-memory representations of vectorized execution framework. mod arrow; -pub use arrow::{to_deltalake_record_batch_with_schema, to_record_batch_with_schema}; +pub use arrow::{ + to_deltalake_record_batch_with_schema, to_iceberg_record_batch_with_schema, + to_record_batch_with_schema, +}; mod bool_array; pub mod bytes_array; mod chrono_array; diff --git a/src/common/src/array/stream_chunk.rs b/src/common/src/array/stream_chunk.rs index 711f06f754414..0fd5563d4a790 100644 --- a/src/common/src/array/stream_chunk.rs +++ b/src/common/src/array/stream_chunk.rs @@ -303,6 +303,33 @@ impl StreamChunk { } } + /// Remove the adjacent delete-insert if their row value are the same. + pub fn eliminate_adjacent_noop_update(self) -> Self { + let len = self.data_chunk().capacity(); + let mut c: StreamChunkMut = self.into(); + let mut prev_r = None; + for curr in 0..len { + if !c.vis(curr) { + continue; + } + if let Some(prev) = prev_r { + if matches!(c.op(prev), Op::UpdateDelete | Op::Delete) + && matches!(c.op(curr), Op::UpdateInsert | Op::Insert) + && c.row_ref(prev) == c.row_ref(curr) + { + c.set_vis(prev, false); + c.set_vis(curr, false); + prev_r = None; + } else { + prev_r = Some(curr) + } + } else { + prev_r = Some(curr); + } + } + c.into() + } + /// Reorder columns and set visibility. pub fn project_with_vis(&self, indices: &[usize], vis: Bitmap) -> Self { Self { @@ -492,6 +519,18 @@ impl OpRowMutRef<'_> { } impl StreamChunkMut { + pub fn vis(&self, i: usize) -> bool { + self.vis.is_set(i) + } + + pub fn op(&self, i: usize) -> Op { + self.ops.get(i) + } + + pub fn row_ref(&self, i: usize) -> RowRef<'_> { + RowRef::with_columns(self.columns(), i) + } + pub fn set_vis(&mut self, n: usize, val: bool) { self.vis.set(n, val); } @@ -788,6 +827,36 @@ mod tests { "\ +---+---+---+ | - | 2 | | ++---+---+---+" + ); + } + + #[test] + fn test_eliminate_adjacent_noop_update() { + let c = StreamChunk::from_pretty( + " I I + - 1 6 D + - 2 2 + + 2 3 + - 2 3 + + 1 6 + - 1 7 + + 1 10 D + + 1 7 + U- 3 7 + U+ 3 7 + + 2 3", + ); + let c = c.eliminate_adjacent_noop_update(); + assert_eq!( + c.to_pretty().to_string(), + "\ ++---+---+---+ +| - | 2 | 2 | +| + | 2 | 3 | +| - | 2 | 3 | +| + | 1 | 6 | +| + | 2 | 3 | +---+---+---+" ); } diff --git a/src/common/src/array/stream_chunk_builder.rs b/src/common/src/array/stream_chunk_builder.rs index 3c1391af3fa44..2d70c925d696f 100644 --- a/src/common/src/array/stream_chunk_builder.rs +++ b/src/common/src/array/stream_chunk_builder.rs @@ -14,6 +14,7 @@ use crate::array::stream_record::Record; use crate::array::{ArrayBuilderImpl, Op, StreamChunk}; +use crate::buffer::BitmapBuilder; use crate::row::Row; use crate::types::{DataType, DatumRef}; use crate::util::iter_util::ZipEqFast; @@ -26,6 +27,9 @@ pub struct StreamChunkBuilder { /// arrays in the data chunk to build column_builders: Vec, + /// Visibility + vis_builder: BitmapBuilder, + /// Data types of columns data_types: Vec, @@ -61,6 +65,7 @@ impl StreamChunkBuilder { ops, column_builders, data_types, + vis_builder: BitmapBuilder::default(), capacity: chunk_size, size: 0, } @@ -93,18 +98,34 @@ impl StreamChunkBuilder { &mut self, op: Op, iter: impl IntoIterator)>, + ) -> Option { + self.append_iter_inner::(op, iter) + } + + #[must_use] + fn append_iter_inner<'a, const VIS: bool>( + &mut self, + op: Op, + iter: impl IntoIterator)>, ) -> Option { self.ops.push(op); for (i, datum) in iter { self.column_builders[i].append(datum); } + self.vis_builder.append(VIS); self.inc_size() } /// Append a row to the builder, return a chunk if the builder is full. #[must_use] pub fn append_row(&mut self, op: Op, row: impl Row) -> Option { - self.append_iter(op, row.iter().enumerate()) + self.append_iter_inner::(op, row.iter().enumerate()) + } + + /// Append an invisible row to the builder, return a chunk if the builder is full. + #[must_use] + pub fn append_row_invisible(&mut self, op: Op, row: impl Row) -> Option { + self.append_iter_inner::(op, row.iter().enumerate()) } /// Append a record to the builder, return a chunk if the builder is full. @@ -138,9 +159,12 @@ impl StreamChunkBuilder { .map(Into::into) .collect::>(); - Some(StreamChunk::new( + let vis = std::mem::take(&mut self.vis_builder).finish(); + + Some(StreamChunk::with_visibility( std::mem::replace(&mut self.ops, Vec::with_capacity(self.capacity)), new_columns, + vis, )) } } diff --git a/src/common/src/cache.rs b/src/common/src/cache.rs index 3649230d455e5..e1e97891c1304 100644 --- a/src/common/src/cache.rs +++ b/src/common/src/cache.rs @@ -994,12 +994,6 @@ pub enum LookupResult { unsafe impl Send for CacheableEntry {} unsafe impl Sync for CacheableEntry {} -impl CacheableEntry { - pub fn value(&self) -> &T { - unsafe { (*self.handle).get_value() } - } -} - impl Deref for CacheableEntry { type Target = T; @@ -1080,7 +1074,7 @@ mod tests { block_offset.hash(&mut hasher); let h = hasher.finish(); if let Some(block) = cache.lookup(h, &(sst, block_offset)) { - assert_eq!(block.value().offset, block_offset); + assert_eq!(block.offset, block_offset); drop(block); continue; } @@ -1382,7 +1376,7 @@ mod tests { cache.insert("b".to_string(), 0, 1, "v2".to_string(), CachePriority::Low); recv.try_recv().unwrap(); assert!( - matches!(cache.lookup_for_request(0, "b".to_string()), LookupResult::Cached(v) if v.value().eq("v2")) + matches!(cache.lookup_for_request(0, "b".to_string()), LookupResult::Cached(v) if v.eq("v2")) ); } _ => panic!(), diff --git a/src/common/src/cast/mod.rs b/src/common/src/cast/mod.rs index dbe39570e5266..41d3c1c1ceae9 100644 --- a/src/common/src/cast/mod.rs +++ b/src/common/src/cast/mod.rs @@ -20,8 +20,6 @@ type Result = std::result::Result; pub const PARSE_ERROR_STR_TO_BYTEA: &str = "Invalid Bytea syntax"; -const ERROR_INT_TO_TIMESTAMP: &str = "Can't cast negative integer to timestamp"; - /// Parse a string into a bool. /// /// See [`https://www.postgresql.org/docs/9.5/datatype-boolean.html`] @@ -60,15 +58,14 @@ pub fn str_to_bool(input: &str) -> Result { /// This would cause no problem for timestamp in [1973-03-03 09:46:40, 5138-11-16 09:46:40). #[inline] pub fn i64_to_timestamptz(t: i64) -> Result { - const E11: i64 = 100_000_000_000; - const E14: i64 = 100_000_000_000_000; - const E17: i64 = 100_000_000_000_000_000; - match t { + const E11: u64 = 100_000_000_000; + const E14: u64 = 100_000_000_000_000; + const E17: u64 = 100_000_000_000_000_000; + match t.abs_diff(0) { 0..E11 => Ok(Timestamptz::from_secs(t).unwrap()), // s E11..E14 => Ok(Timestamptz::from_millis(t).unwrap()), // ms E14..E17 => Ok(Timestamptz::from_micros(t)), // us E17.. => Ok(Timestamptz::from_micros(t / 1000)), // ns - _ => Err(ERROR_INT_TO_TIMESTAMP.to_string()), } } @@ -193,8 +190,19 @@ pub fn parse_bytes_traditional(s: &str) -> Result> { #[cfg(test)] mod tests { + use chrono::{DateTime, Utc}; + use super::*; + #[test] + fn test_negative_int_to_timestamptz() { + let x = i64_to_timestamptz(-2208988800000000000) + .unwrap() + .to_datetime_utc(); + let ans: DateTime = "1900-01-01T00:00:00Z".parse().unwrap(); + assert_eq!(x, ans); + } + #[test] fn test_bytea() { use crate::types::ToText; diff --git a/src/common/src/catalog/column.rs b/src/common/src/catalog/column.rs index 7be23eb3a0b19..82d2f22f41cb4 100644 --- a/src/common/src/catalog/column.rs +++ b/src/common/src/catalog/column.rs @@ -18,7 +18,7 @@ use itertools::Itertools; use risingwave_pb::expr::ExprNode; use risingwave_pb::plan_common::column_desc::GeneratedOrDefaultColumn; use risingwave_pb::plan_common::{ - AdditionalColumnType, ColumnDescVersion, PbColumnCatalog, PbColumnDesc, + AdditionalColumn, ColumnDescVersion, PbColumnCatalog, PbColumnDesc, }; use super::row_id_column_desc; @@ -103,7 +103,7 @@ pub struct ColumnDesc { pub type_name: String, pub generated_or_default_column: Option, pub description: Option, - pub additional_column_type: AdditionalColumnType, + pub additional_column: AdditionalColumn, pub version: ColumnDescVersion, } @@ -117,7 +117,7 @@ impl ColumnDesc { type_name: String::new(), generated_or_default_column: None, description: None, - additional_column_type: AdditionalColumnType::Normal, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -131,7 +131,7 @@ impl ColumnDesc { type_name: String::new(), generated_or_default_column: None, description: None, - additional_column_type: AdditionalColumnType::Normal, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -140,7 +140,7 @@ impl ColumnDesc { name: impl Into, column_id: ColumnId, data_type: DataType, - additional_column_type: AdditionalColumnType, + additional_column_type: AdditionalColumn, ) -> ColumnDesc { ColumnDesc { data_type, @@ -150,7 +150,7 @@ impl ColumnDesc { type_name: String::new(), generated_or_default_column: None, description: None, - additional_column_type, + additional_column: additional_column_type, version: ColumnDescVersion::Pr13707, } } @@ -170,7 +170,8 @@ impl ColumnDesc { type_name: self.type_name.clone(), generated_or_default_column: self.generated_or_default_column.clone(), description: self.description.clone(), - additional_column_type: self.additional_column_type as i32, + additional_column_type: 0, // deprecated + additional_column: Some(self.additional_column.clone()), version: self.version as i32, } } @@ -198,7 +199,7 @@ impl ColumnDesc { type_name: "".to_string(), generated_or_default_column: None, description: None, - additional_column_type: AdditionalColumnType::Normal, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -221,7 +222,7 @@ impl ColumnDesc { type_name: type_name.to_string(), generated_or_default_column: None, description: None, - additional_column_type: AdditionalColumnType::Normal, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -239,7 +240,7 @@ impl ColumnDesc { type_name: field.type_name.clone(), description: None, generated_or_default_column: None, - additional_column_type: AdditionalColumnType::Normal, + additional_column: AdditionalColumn { column_type: None }, version: ColumnDescVersion::Pr13707, } } @@ -265,7 +266,10 @@ impl ColumnDesc { impl From for ColumnDesc { fn from(prost: PbColumnDesc) -> Self { - let additional_column_type = prost.additional_column_type(); + let additional_column = prost + .get_additional_column() + .unwrap_or(&AdditionalColumn { column_type: None }) + .clone(); let version = prost.version(); let field_descs: Vec = prost .field_descs @@ -280,7 +284,7 @@ impl From for ColumnDesc { field_descs, generated_or_default_column: prost.generated_or_default_column, description: prost.description.clone(), - additional_column_type, + additional_column, version, } } @@ -302,7 +306,8 @@ impl From<&ColumnDesc> for PbColumnDesc { type_name: c.type_name.clone(), generated_or_default_column: c.generated_or_default_column.clone(), description: c.description.clone(), - additional_column_type: c.additional_column_type as i32, + additional_column_type: 0, // deprecated + additional_column: c.additional_column.clone().into(), version: c.version as i32, } } diff --git a/src/common/src/catalog/internal_table.rs b/src/common/src/catalog/internal_table.rs index 2dfe23949b1ea..1e991db6975f1 100644 --- a/src/common/src/catalog/internal_table.rs +++ b/src/common/src/catalog/internal_table.rs @@ -12,8 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::any::type_name; +use std::fmt::Debug; use std::sync::LazyLock; +use anyhow::anyhow; use itertools::Itertools; use regex::Regex; @@ -40,22 +43,29 @@ pub fn valid_table_name(table_name: &str) -> bool { !INTERNAL_TABLE_NAME.is_match(table_name) } -pub fn get_dist_key_in_pk_indices(dist_key_indices: &[usize], pk_indices: &[usize]) -> Vec { - let dist_key_in_pk_indices = dist_key_indices +pub fn get_dist_key_in_pk_indices>( + dist_key_indices: &[I], + pk_indices: &[I], +) -> anyhow::Result> { + dist_key_indices .iter() .map(|&di| { pk_indices .iter() .position(|&pi| di == pi) - .unwrap_or_else(|| { - panic!( + .ok_or_else(|| { + anyhow!( "distribution key {:?} must be a subset of primary key {:?}", - dist_key_indices, pk_indices + dist_key_indices, + pk_indices ) }) + .map(|idx| match O::try_from(idx) { + Ok(idx) => idx, + Err(_) => unreachable!("failed to cast {} to {}", idx, type_name::()), + }) }) - .collect_vec(); - dist_key_in_pk_indices + .try_collect() } /// Get distribution key start index in pk, and return None if `dist_key_in_pk_indices` is not empty diff --git a/src/common/src/catalog/mod.rs b/src/common/src/catalog/mod.rs index 14d44c9c0bd2d..5e11860ca180b 100644 --- a/src/common/src/catalog/mod.rs +++ b/src/common/src/catalog/mod.rs @@ -19,7 +19,6 @@ mod physical_table; mod schema; pub mod test_utils; -use std::collections::HashMap; use std::sync::Arc; use async_trait::async_trait; @@ -31,11 +30,10 @@ pub use physical_table::*; use risingwave_pb::catalog::HandleConflictBehavior as PbHandleConflictBehavior; use risingwave_pb::plan_common::ColumnDescVersion; pub use schema::{test_utils as schema_test_utils, Field, FieldDisplay, Schema}; -use thiserror_ext::AsReport; +use crate::array::DataChunk; pub use crate::constants::hummock; use crate::error::BoxedError; -use crate::row::OwnedRow; use crate::types::DataType; /// The global version of the catalog. @@ -65,6 +63,8 @@ pub const DEFAULT_SUPER_USER_FOR_PG_ID: u32 = 2; pub const NON_RESERVED_USER_ID: i32 = 11; pub const NON_RESERVED_SYS_CATALOG_ID: i32 = 1001; +pub const OBJECT_ID_PLACEHOLDER: u32 = u32::MAX - 1; + pub const SYSTEM_SCHEMAS: [&str; 3] = [ PG_CATALOG_SCHEMA_NAME, INFORMATION_SCHEMA_SCHEMA_NAME, @@ -146,7 +146,7 @@ pub fn cdc_table_name_column_desc() -> ColumnDesc { /// The local system catalog reader in the frontend node. #[async_trait] pub trait SysCatalogReader: Sync + Send + 'static { - async fn read_table(&self, table_id: &TableId) -> Result, BoxedError>; + async fn read_table(&self, table_id: &TableId) -> Result; } pub type SysCatalogReaderRef = Arc; @@ -164,7 +164,7 @@ impl DatabaseId { pub fn placeholder() -> Self { DatabaseId { - database_id: u32::MAX - 1, + database_id: OBJECT_ID_PLACEHOLDER, } } } @@ -200,7 +200,7 @@ impl SchemaId { pub fn placeholder() -> Self { SchemaId { - schema_id: u32::MAX - 1, + schema_id: OBJECT_ID_PLACEHOLDER, } } } @@ -237,7 +237,7 @@ impl TableId { /// Sometimes the id field is filled later, we use this value for better debugging. pub const fn placeholder() -> Self { TableId { - table_id: u32::MAX - 1, + table_id: OBJECT_ID_PLACEHOLDER, } } @@ -271,46 +271,24 @@ pub struct TableOption { impl From<&risingwave_pb::hummock::TableOption> for TableOption { fn from(table_option: &risingwave_pb::hummock::TableOption) -> Self { - let retention_seconds = - if table_option.retention_seconds == hummock::TABLE_OPTION_DUMMY_RETENTION_SECOND { - None - } else { - Some(table_option.retention_seconds) - }; - - Self { retention_seconds } + Self { + retention_seconds: table_option.retention_seconds, + } } } impl From<&TableOption> for risingwave_pb::hummock::TableOption { fn from(table_option: &TableOption) -> Self { Self { - retention_seconds: table_option - .retention_seconds - .unwrap_or(hummock::TABLE_OPTION_DUMMY_RETENTION_SECOND), + retention_seconds: table_option.retention_seconds, } } } impl TableOption { - pub fn build_table_option(table_properties: &HashMap) -> Self { + pub fn new(retention_seconds: Option) -> Self { // now we only support ttl for TableOption - let mut result = TableOption::default(); - if let Some(ttl_string) = table_properties.get(hummock::PROPERTIES_RETENTION_SECOND_KEY) { - match ttl_string.trim().parse::() { - Ok(retention_seconds_u32) => result.retention_seconds = Some(retention_seconds_u32), - Err(e) => { - tracing::info!( - error = %e.as_report(), - "build_table_option parse option ttl_string {}", - ttl_string, - ); - result.retention_seconds = None; - } - }; - } - - result + TableOption { retention_seconds } } } @@ -328,7 +306,7 @@ impl IndexId { /// Sometimes the id field is filled later, we use this value for better debugging. pub const fn placeholder() -> Self { IndexId { - index_id: u32::MAX - 1, + index_id: OBJECT_ID_PLACEHOLDER, } } @@ -357,7 +335,7 @@ impl FunctionId { } pub const fn placeholder() -> Self { - FunctionId(u32::MAX - 1) + FunctionId(OBJECT_ID_PLACEHOLDER) } pub fn function_id(&self) -> u32 { @@ -396,7 +374,7 @@ impl UserId { pub const fn placeholder() -> Self { UserId { - user_id: u32::MAX - 1, + user_id: OBJECT_ID_PLACEHOLDER, } } } @@ -428,7 +406,7 @@ impl ConnectionId { } pub const fn placeholder() -> Self { - ConnectionId(u32::MAX - 1) + ConnectionId(OBJECT_ID_PLACEHOLDER) } pub fn connection_id(&self) -> u32 { diff --git a/src/common/src/catalog/physical_table.rs b/src/common/src/catalog/physical_table.rs index 92c71d37e2aa1..5ed66b98de5c6 100644 --- a/src/common/src/catalog/physical_table.rs +++ b/src/common/src/catalog/physical_table.rs @@ -14,16 +14,13 @@ use std::collections::HashMap; -use anyhow::anyhow; use fixedbitset::FixedBitSet; -use itertools::Itertools; use risingwave_pb::catalog::Table; use risingwave_pb::common::PbColumnOrder; use risingwave_pb::plan_common::StorageTableDesc; use super::{ColumnDesc, ColumnId, TableId}; -use crate::catalog::hummock::TABLE_OPTION_DUMMY_RETENTION_SECOND; -use crate::catalog::TableOption; +use crate::catalog::get_dist_key_in_pk_indices; use crate::util::sort_util::ColumnOrder; /// Includes necessary information for compute node to access data of the table. @@ -49,7 +46,8 @@ pub struct TableDesc { /// Whether the table source is append-only pub append_only: bool, - pub retention_seconds: u32, + // TTL of the record in the table, to ensure the consistency with other tables in the streaming plan, it only applies to append-only tables. + pub retention_seconds: Option, pub value_indices: Vec, @@ -100,22 +98,7 @@ impl TableDesc { .map(|i| i as u32); let dist_key_in_pk_indices = if vnode_col_idx_in_pk.is_none() { - dist_key_indices - .iter() - .map(|&di| { - pk_indices - .iter() - .position(|&pi| di == pi) - .ok_or_else(|| { - anyhow!( - "distribution key {:?} must be a subset of primary key {:?}", - dist_key_indices, - pk_indices - ) - }) - .map(|d| d as u32) - }) - .try_collect()? + get_dist_key_in_pk_indices(&dist_key_indices, &pk_indices)? } else { Vec::new() }; @@ -143,7 +126,6 @@ impl TableDesc { } pub fn from_pb_table(table: &Table) -> Self { - let table_options = TableOption::build_table_option(&table.properties); Self { table_id: TableId::new(table.id), pk: table.pk.iter().map(ColumnOrder::from_protobuf).collect(), @@ -156,9 +138,7 @@ impl TableDesc { stream_key: table.stream_key.iter().map(|i| *i as _).collect(), vnode_col_index: table.vnode_col_index.map(|i| i as _), append_only: table.append_only, - retention_seconds: table_options - .retention_seconds - .unwrap_or(TABLE_OPTION_DUMMY_RETENTION_SECOND), + retention_seconds: table.retention_seconds, value_indices: table.value_indices.iter().map(|i| *i as _).collect(), read_prefix_len_hint: table.read_prefix_len_hint as _, watermark_columns: table.watermark_indices.iter().map(|i| *i as _).collect(), diff --git a/src/common/src/catalog/test_utils.rs b/src/common/src/catalog/test_utils.rs index db8efb278dd83..ae87b3a881f84 100644 --- a/src/common/src/catalog/test_utils.rs +++ b/src/common/src/catalog/test_utils.rs @@ -15,7 +15,7 @@ use itertools::Itertools; use risingwave_pb::data::data_type::TypeName; use risingwave_pb::data::DataType; -use risingwave_pb::plan_common::{AdditionalColumnType, ColumnDesc, ColumnDescVersion}; +use risingwave_pb::plan_common::{AdditionalColumn, ColumnDesc, ColumnDescVersion}; pub trait ColumnDescTestExt { /// Create a [`ColumnDesc`] with the given name and type. @@ -35,7 +35,7 @@ impl ColumnDescTestExt for ColumnDesc { column_type: Some(data_type), column_id, name: name.to_string(), - additional_column_type: AdditionalColumnType::Normal as i32, + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, ..Default::default() } @@ -60,7 +60,8 @@ impl ColumnDescTestExt for ColumnDesc { field_descs: fields, generated_or_default_column: None, description: None, - additional_column_type: AdditionalColumnType::Normal as i32, + additional_column_type: 0, // deprecated + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, } } diff --git a/src/common/src/config.rs b/src/common/src/config.rs index 49b0453334c17..00d36ced99ff8 100644 --- a/src/common/src/config.rs +++ b/src/common/src/config.rs @@ -25,12 +25,14 @@ use std::num::NonZeroUsize; use anyhow::Context; use clap::ValueEnum; use educe::Educe; +use risingwave_common_proc_macro::ConfigDoc; pub use risingwave_common_proc_macro::OverrideConfig; use risingwave_pb::meta::SystemParams; use serde::{Deserialize, Serialize, Serializer}; use serde_default::DefaultFromSerde; use serde_json::Value; +use crate::for_all_params; use crate::hash::VirtualNode; /// Use the maximum value for HTTP/2 connection window size to avoid deadlock among multiplexed @@ -131,29 +133,36 @@ impl OverrideConfig for NoOverride { /// [`RwConfig`] corresponds to the whole config file `risingwave.toml`. Each field corresponds to a /// section. -#[derive(Educe, Clone, Serialize, Deserialize, Default)] +#[derive(Educe, Clone, Serialize, Deserialize, Default, ConfigDoc)] #[educe(Debug)] pub struct RwConfig { #[serde(default)] + #[config_doc(nested)] pub server: ServerConfig, #[serde(default)] + #[config_doc(nested)] pub meta: MetaConfig, #[serde(default)] + #[config_doc(nested)] pub batch: BatchConfig, #[serde(default)] + #[config_doc(nested)] pub streaming: StreamingConfig, #[serde(default)] + #[config_doc(nested)] pub storage: StorageConfig, #[serde(default)] #[educe(Debug(ignore))] + #[config_doc(nested)] pub system: SystemConfig, #[serde(flatten)] + #[config_doc(omitted)] pub unrecognized: Unrecognized, } @@ -166,10 +175,11 @@ pub enum MetaBackend { #[default] Mem, Etcd, + Sql, } /// The section `[meta]` in `risingwave.toml`. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct MetaConfig { /// Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they /// are dangling. @@ -202,6 +212,14 @@ pub struct MetaConfig { #[serde(default = "default::meta::hummock_version_checkpoint_interval_sec")] pub hummock_version_checkpoint_interval_sec: u64, + /// If enabled, SSTable object file and version delta will be retained. + /// + /// SSTable object file need to be deleted via full GC. + /// + /// version delta need to be manually deleted. + #[serde(default = "default::meta::enable_hummock_data_archive")] + pub enable_hummock_data_archive: bool, + /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint /// attempt is rejected. #[serde(default = "default::meta::min_delta_log_num_for_hummock_version_checkpoint")] @@ -215,13 +233,21 @@ pub struct MetaConfig { #[serde(default)] pub disable_recovery: bool, - /// Whether to enable scale-in when recovery. + /// Whether to disable adaptive-scaling feature. #[serde(default)] - pub enable_scale_in_when_recovery: bool, + pub disable_automatic_parallelism_control: bool, - /// Whether to enable auto-scaling feature. - #[serde(default)] - pub enable_automatic_parallelism_control: bool, + /// The number of streaming jobs per scaling operation. + #[serde(default = "default::meta::parallelism_control_batch_size")] + pub parallelism_control_batch_size: usize, + + /// The period of parallelism control trigger. + #[serde(default = "default::meta::parallelism_control_trigger_period_sec")] + pub parallelism_control_trigger_period_sec: u64, + + /// The first delay of parallelism control. + #[serde(default = "default::meta::parallelism_control_trigger_first_delay_sec")] + pub parallelism_control_trigger_first_delay_sec: u64, #[serde(default = "default::meta::meta_leader_lease_secs")] pub meta_leader_lease_secs: u64, @@ -305,6 +331,7 @@ pub struct MetaConfig { pub compaction_task_max_progress_interval_secs: u64, #[serde(default)] + #[config_doc(nested)] pub compaction_config: CompactionConfig, #[serde(default = "default::meta::hybird_partition_vnode_count")] @@ -316,6 +343,7 @@ pub struct MetaConfig { pub event_log_channel_max_size: u32, #[serde(default, with = "meta_prefix")] + #[config_doc(omitted)] pub developer: MetaDeveloperConfig, } @@ -375,9 +403,9 @@ impl<'de> Deserialize<'de> for DefaultParallelism { VirtualNode::COUNT )))? } else { - NonZeroUsize::new(i) - .context("default parallelism should be greater than 0") - .map_err(|e| serde::de::Error::custom(e.to_string()))? + NonZeroUsize::new(i).ok_or_else(|| { + serde::de::Error::custom("default parallelism should be greater than 0") + })? })), } } @@ -386,7 +414,7 @@ impl<'de> Deserialize<'de> for DefaultParallelism { /// The subsections `[meta.developer]`. /// /// It is put at [`MetaConfig::developer`]. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct MetaDeveloperConfig { /// The number of traces to be cached in-memory by the tracing collector /// embedded in the meta node. @@ -397,10 +425,16 @@ pub struct MetaDeveloperConfig { /// in the meta node. #[serde(default = "default::developer::meta_cached_traces_memory_limit_bytes")] pub cached_traces_memory_limit_bytes: usize, + + /// Compaction picker config + #[serde(default = "default::developer::enable_trivial_move")] + pub enable_trivial_move: bool, + #[serde(default = "default::developer::enable_check_task_level_overlap")] + pub enable_check_task_level_overlap: bool, } /// The section `[server]` in `risingwave.toml`. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct ServerConfig { /// The interval for periodic heartbeat from worker to the meta service. #[serde(default = "default::server::heartbeat_interval_ms")] @@ -425,11 +459,12 @@ pub struct ServerConfig { pub grpc_max_reset_stream: u32, #[serde(default, flatten)] + #[config_doc(omitted)] pub unrecognized: Unrecognized, } /// The section `[batch]` in `risingwave.toml`. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct BatchConfig { /// The thread number of the batch task runtime in the compute node. The default value is /// decided by `tokio`. @@ -437,6 +472,7 @@ pub struct BatchConfig { pub worker_threads_num: Option, #[serde(default, with = "batch_prefix")] + #[config_doc(omitted)] pub developer: BatchDeveloperConfig, #[serde(default)] @@ -450,11 +486,16 @@ pub struct BatchConfig { pub statement_timeout_in_sec: u32, #[serde(default, flatten)] + #[config_doc(omitted)] pub unrecognized: Unrecognized, + + #[serde(default = "default::batch::frontend_compute_runtime_worker_threads")] + /// frontend compute runtime worker threads + pub frontend_compute_runtime_worker_threads: usize, } /// The section `[streaming]` in `risingwave.toml`. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct StreamingConfig { /// The maximum number of barriers in-flight in the compute nodes. #[serde(default = "default::streaming::in_flight_barrier_nums")] @@ -470,6 +511,7 @@ pub struct StreamingConfig { pub async_stack_trace: AsyncStackTraceOption, #[serde(default, with = "streaming_prefix")] + #[config_doc(omitted)] pub developer: StreamingDeveloperConfig, /// Max unique user stream errors per actor @@ -477,6 +519,7 @@ pub struct StreamingConfig { pub unique_user_stream_errors: usize, #[serde(default, flatten)] + #[config_doc(omitted)] pub unrecognized: Unrecognized, } @@ -517,7 +560,7 @@ impl PartialOrd for MetricLevel { } /// The section `[storage]` in `risingwave.toml`. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct StorageConfig { /// parallelism while syncing share buffers into L0 SST. Should NOT be 0. #[serde(default = "default::storage::share_buffers_sync_parallelism")] @@ -624,8 +667,8 @@ pub struct StorageConfig { pub compactor_max_sst_size: u64, #[serde(default = "default::storage::enable_fast_compaction")] pub enable_fast_compaction: bool, - #[serde(default = "default::storage::check_fast_compaction_result")] - pub check_fast_compaction_result: bool, + #[serde(default = "default::storage::check_compaction_result")] + pub check_compaction_result: bool, #[serde(default = "default::storage::max_preload_io_retry_times")] pub max_preload_io_retry_times: usize, @@ -636,6 +679,7 @@ pub struct StorageConfig { pub compactor_fast_max_compact_task_size: u64, #[serde(default, flatten)] + #[config_doc(omitted)] pub unrecognized: Unrecognized, /// The spill threshold for mem table. @@ -646,7 +690,7 @@ pub struct StorageConfig { pub object_store: ObjectStoreConfig, } -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct CacheRefillConfig { /// SSTable levels to refill. #[serde(default = "default::cache_refill::data_refill_levels")] @@ -679,13 +723,14 @@ pub struct CacheRefillConfig { pub recent_filter_rotate_interval_ms: usize, #[serde(default, flatten)] + #[config_doc(omitted)] pub unrecognized: Unrecognized, } /// The subsection `[storage.data_file_cache]` and `[storage.meta_file_cache]` in `risingwave.toml`. /// /// It's put at [`StorageConfig::data_file_cache`] and [`StorageConfig::meta_file_cache`]. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct FileCacheConfig { #[serde(default = "default::file_cache::dir")] pub dir: String, @@ -730,6 +775,7 @@ pub struct FileCacheConfig { pub compression: String, #[serde(default, flatten)] + #[config_doc(omitted)] pub unrecognized: Unrecognized, } @@ -766,7 +812,7 @@ pub enum CompactorMode { Shared, } -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct HeapProfilingConfig { /// Enable to auto dump heap profile when memory usage is high #[serde(default = "default::heap_profiling::enable_auto")] @@ -784,7 +830,7 @@ pub struct HeapProfilingConfig { /// The subsections `[streaming.developer]`. /// /// It is put at [`StreamingConfig::developer`]. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct StreamingDeveloperConfig { /// Set to true to enable per-executor row count metrics. This will produce a lot of timeseries /// and might affect the prometheus performance. If you only need actor input and output @@ -839,7 +885,7 @@ pub struct StreamingDeveloperConfig { /// The subsections `[batch.developer]`. /// /// It is put at [`BatchConfig::developer`]. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct BatchDeveloperConfig { /// The capacity of the chunks in the channel that connects between `ConnectorSource` and /// `SourceExecutor`. @@ -854,65 +900,27 @@ pub struct BatchDeveloperConfig { #[serde(default = "default::developer::batch_chunk_size")] pub chunk_size: usize, } -/// The section `[system]` in `risingwave.toml`. All these fields are used to initialize the system -/// parameters persisted in Meta store. Most fields are for testing purpose only and should not be -/// documented. -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] -pub struct SystemConfig { - /// The interval of periodic barrier. - #[serde(default = "default::system::barrier_interval_ms")] - pub barrier_interval_ms: Option, - - /// There will be a checkpoint for every n barriers - #[serde(default = "default::system::checkpoint_frequency")] - pub checkpoint_frequency: Option, - - /// Target size of the Sstable. - #[serde(default = "default::system::sstable_size_mb")] - pub sstable_size_mb: Option, - - #[serde(default = "default::system::parallel_compact_size_mb")] - pub parallel_compact_size_mb: Option, - - /// Size of each block in bytes in SST. - #[serde(default = "default::system::block_size_kb")] - pub block_size_kb: Option, - - /// False positive probability of bloom filter. - #[serde(default = "default::system::bloom_false_positive")] - pub bloom_false_positive: Option, - #[serde(default = "default::system::state_store")] - pub state_store: Option, - - /// Remote directory for storing data and metadata objects. - #[serde(default = "default::system::data_directory")] - pub data_directory: Option, - - /// Remote storage url for storing snapshots. - #[serde(default = "default::system::backup_storage_url")] - pub backup_storage_url: Option, - - /// Remote directory for storing snapshots. - #[serde(default = "default::system::backup_storage_directory")] - pub backup_storage_directory: Option, - - /// Max number of concurrent creating streaming jobs. - #[serde(default = "default::system::max_concurrent_creating_streaming_jobs")] - pub max_concurrent_creating_streaming_jobs: Option, - - /// Whether to pause all data sources on next bootstrap. - #[serde(default = "default::system::pause_on_next_bootstrap")] - pub pause_on_next_bootstrap: Option, - - #[serde(default = "default::system::wasm_storage_url")] - pub wasm_storage_url: Option, - - /// Whether to enable distributed tracing. - #[serde(default = "default::system::enable_tracing")] - pub enable_tracing: Option, +macro_rules! define_system_config { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr, $doc:literal, $($rest:tt)* },)*) => { + paste::paste!( + /// The section `[system]` in `risingwave.toml`. All these fields are used to initialize the system + /// parameters persisted in Meta store. Most fields are for testing purpose only and should not be + /// documented. + #[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] + pub struct SystemConfig { + $( + #[doc = $doc] + #[serde(default = "default::system::" $field "_opt")] + pub $field: Option<$type>, + )* + } + ); + }; } +for_all_params!(define_system_config); + /// The subsections `[storage.object_store]`. #[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] pub struct ObjectStoreConfig { @@ -946,28 +954,63 @@ pub struct S3ObjectStoreConfig { pub object_store_req_retry_max_delay_ms: u64, #[serde(default = "default::object_store_config::s3::object_store_req_retry_max_attempts")] pub object_store_req_retry_max_attempts: usize, + /// For backwards compatibility, users should use `S3ObjectStoreDeveloperConfig` instead. + #[serde( + default = "default::object_store_config::s3::developer::object_store_retry_unknown_service_error" + )] + pub retry_unknown_service_error: bool, + #[serde(default)] + pub developer: S3ObjectStoreDeveloperConfig, +} + +/// The subsections `[storage.object_store.s3.developer]`. +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +pub struct S3ObjectStoreDeveloperConfig { + /// Whether to retry s3 sdk error from which no error metadata is provided. + #[serde( + default = "default::object_store_config::s3::developer::object_store_retry_unknown_service_error" + )] + pub object_store_retry_unknown_service_error: bool, + /// An array of error codes that should be retried. + /// e.g. `["SlowDown", "TooManyRequests"]` + #[serde( + default = "default::object_store_config::s3::developer::object_store_retryable_service_error_codes" + )] + pub object_store_retryable_service_error_codes: Vec, } impl SystemConfig { #![allow(deprecated)] pub fn into_init_system_params(self) -> SystemParams { - SystemParams { - barrier_interval_ms: self.barrier_interval_ms, - checkpoint_frequency: self.checkpoint_frequency, - sstable_size_mb: self.sstable_size_mb, - parallel_compact_size_mb: self.parallel_compact_size_mb, - block_size_kb: self.block_size_kb, - bloom_false_positive: self.bloom_false_positive, - state_store: self.state_store, - data_directory: self.data_directory, - backup_storage_url: self.backup_storage_url, - backup_storage_directory: self.backup_storage_directory, - max_concurrent_creating_streaming_jobs: self.max_concurrent_creating_streaming_jobs, - pause_on_next_bootstrap: self.pause_on_next_bootstrap, - wasm_storage_url: self.wasm_storage_url, - enable_tracing: self.enable_tracing, - telemetry_enabled: None, // deprecated + macro_rules! fields { + ($({ $field:ident, $($rest:tt)* },)*) => { + SystemParams { + $($field: self.$field,)* + ..Default::default() // deprecated fields + } + }; } + + let mut system_params = for_all_params!(fields); + + // Initialize backup_storage_url and backup_storage_directory if not set. + if let Some(state_store) = &system_params.state_store + && let Some(data_directory) = &system_params.data_directory + { + if system_params.backup_storage_url.is_none() { + if let Some(hummock_state_store) = state_store.strip_prefix("hummock+") { + system_params.backup_storage_url = Some(hummock_state_store.to_owned()); + } else { + system_params.backup_storage_url = Some("memory".to_string()); + } + tracing::info!("initialize backup_storage_url based on state_store"); + } + if system_params.backup_storage_directory.is_none() { + system_params.backup_storage_directory = Some(format!("{data_directory}/backup")); + tracing::info!("initialize backup_storage_directory based on data_directory"); + } + } + system_params } } @@ -1003,6 +1046,10 @@ pub mod default { 30 } + pub fn enable_hummock_data_archive() -> bool { + false + } + pub fn min_delta_log_num_for_hummock_version_checkpoint() -> u64 { 10 } @@ -1086,6 +1133,18 @@ pub mod default { pub fn event_log_channel_max_size() -> u32 { 10 } + + pub fn parallelism_control_batch_size() -> usize { + 10 + } + + pub fn parallelism_control_trigger_period_sec() -> u64 { + 10 + } + + pub fn parallelism_control_trigger_first_delay_sec() -> u64 { + 30 + } } pub mod server { @@ -1210,7 +1269,7 @@ pub mod default { false } - pub fn check_fast_compaction_result() -> bool { + pub fn check_compaction_result() -> bool { false } @@ -1218,7 +1277,7 @@ pub mod default { 3 } pub fn mem_table_spill_threshold() -> usize { - 0 // disable + 4 << 20 } pub fn compactor_fast_max_compact_delete_ratio() -> u32 { @@ -1411,12 +1470,18 @@ pub mod default { pub fn stream_hash_agg_max_dirty_groups_heap_size() -> usize { 64 << 20 // 64MB } - } - pub mod system { - pub use crate::system_param::default::*; + pub fn enable_trivial_move() -> bool { + true + } + + pub fn enable_check_task_level_overlap() -> bool { + false + } } + pub use crate::system_param::default as system; + pub mod batch { pub fn enable_barrier_read() -> bool { false @@ -1426,6 +1491,10 @@ pub mod default { // 1 hour 60 * 60 } + + pub fn frontend_compute_runtime_worker_threads() -> usize { + 4 + } } pub mod compaction_config { @@ -1434,15 +1503,15 @@ pub mod default { const DEFAULT_MAX_BYTES_FOR_LEVEL_BASE: u64 = 512 * 1024 * 1024; // 512MB // decrease this configure when the generation of checkpoint barrier is not frequent. - const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 6; + const DEFAULT_TIER_COMPACT_TRIGGER_NUMBER: u64 = 12; const DEFAULT_TARGET_FILE_SIZE_BASE: u64 = 32 * 1024 * 1024; // 32MB const DEFAULT_MAX_SUB_COMPACTION: u32 = 4; const DEFAULT_LEVEL_MULTIPLIER: u64 = 5; const DEFAULT_MAX_SPACE_RECLAIM_BYTES: u64 = 512 * 1024 * 1024; // 512MB; const DEFAULT_LEVEL0_STOP_WRITE_THRESHOLD_SUB_LEVEL_NUMBER: u64 = 300; - const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 96; + const DEFAULT_MAX_COMPACTION_FILE_COUNT: u64 = 100; const DEFAULT_MIN_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 3; - const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 6; + const DEFAULT_MIN_OVERLAPPING_SUB_LEVEL_COMPACT_LEVEL_COUNT: u32 = 12; const DEFAULT_TOMBSTONE_RATIO_PERCENT: u32 = 40; const DEFAULT_EMERGENCY_PICKER: bool = true; @@ -1548,6 +1617,16 @@ pub mod default { pub fn object_store_req_retry_max_attempts() -> usize { DEFAULT_RETRY_MAX_ATTEMPTS } + + pub mod developer { + pub fn object_store_retry_unknown_service_error() -> bool { + false + } + + pub fn object_store_retryable_service_error_codes() -> Vec { + vec!["SlowDown".into(), "TooManyRequests".into()] + } + } } } } @@ -1603,7 +1682,7 @@ pub fn extract_storage_memory_config(s: &RwConfig) -> StorageMemoryConfig { } } -#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde)] +#[derive(Clone, Debug, Serialize, Deserialize, DefaultFromSerde, ConfigDoc)] pub struct CompactionConfig { #[serde(default = "default::compaction_config::max_bytes_for_level_base")] pub max_bytes_for_level_base: u64, @@ -1641,6 +1720,8 @@ pub struct CompactionConfig { #[cfg(test)] mod tests { + use std::collections::BTreeMap; + use super::*; /// This test ensures that `config/example.toml` is up-to-date with the default values specified @@ -1656,5 +1737,89 @@ mod tests { let expected = format!("{HEADER}\n\n{default}"); actual.assert_eq(&expected); + + let expected = rw_config_to_markdown(); + let actual = expect_test::expect_file!["../../config/docs.md"]; + actual.assert_eq(&expected); + } + + #[derive(Debug)] + struct ConfigItemDoc { + desc: String, + default: String, + } + + fn rw_config_to_markdown() -> String { + let mut config_rustdocs = BTreeMap::>::new(); + RwConfig::config_docs("".to_string(), &mut config_rustdocs); + + // Section -> Config Name -> ConfigItemDoc + let mut configs: BTreeMap> = config_rustdocs + .into_iter() + .map(|(k, v)| { + let docs: BTreeMap = v + .into_iter() + .map(|(name, desc)| { + ( + name, + ConfigItemDoc { + desc, + default: "".to_string(), // unset + }, + ) + }) + .collect(); + (k, docs) + }) + .collect(); + + let toml_doc: BTreeMap = + toml::from_str(&toml::to_string(&RwConfig::default()).unwrap()).unwrap(); + toml_doc.into_iter().for_each(|(name, value)| { + set_default_values("".to_string(), name, value, &mut configs); + }); + + let mut markdown = "# RisingWave System Configurations\n\n".to_string() + + "This page is automatically generated by `./risedev generate-example-config`\n"; + for (section, configs) in configs { + if configs.is_empty() { + continue; + } + markdown.push_str(&format!("\n## {}\n\n", section)); + markdown.push_str("| Config | Description | Default |\n"); + markdown.push_str("|--------|-------------|---------|\n"); + for (config, doc) in configs { + markdown.push_str(&format!( + "| {} | {} | {} |\n", + config, doc.desc, doc.default + )); + } + } + markdown + } + + fn set_default_values( + section: String, + name: String, + value: toml::Value, + configs: &mut BTreeMap>, + ) { + // Set the default value if it's a config name-value pair, otherwise it's a sub-section (Table) that should be recursively processed. + if let toml::Value::Table(table) = value { + let section_configs: BTreeMap = + table.clone().into_iter().collect(); + let sub_section = if section.is_empty() { + name + } else { + format!("{}.{}", section, name) + }; + section_configs + .into_iter() + .for_each(|(k, v)| set_default_values(sub_section.clone(), k, v, configs)) + } else if let Some(t) = configs.get_mut(§ion) { + if let Some(item_doc) = t.get_mut(&name) { + item_doc.default = format!("{}", value); + } + } } } diff --git a/src/common/src/constants.rs b/src/common/src/constants.rs index 10abc1a7cf0eb..9bdce7b746af7 100644 --- a/src/common/src/constants.rs +++ b/src/common/src/constants.rs @@ -30,32 +30,67 @@ pub mod hummock { flag.bits() } } - - pub const TABLE_OPTION_DUMMY_RETENTION_SECOND: u32 = 0; - pub const PROPERTIES_RETENTION_SECOND_KEY: &str = "retention_seconds"; } pub mod log_store { use crate::types::DataType; + use crate::util::sort_util::OrderType; pub const EPOCH_COLUMN_NAME: &str = "kv_log_store_epoch"; pub const SEQ_ID_COLUMN_NAME: &str = "kv_log_store_seq_id"; pub const ROW_OP_COLUMN_NAME: &str = "kv_log_store_row_op"; - - /// `epoch`, `seq_id`, `row_op` - pub const KV_LOG_STORE_PREDEFINED_COLUMNS: [(&str, DataType); 3] = [ - (EPOCH_COLUMN_NAME, EPOCH_COLUMN_TYPE), - (SEQ_ID_COLUMN_NAME, SEQ_ID_COLUMN_TYPE), - (ROW_OP_COLUMN_NAME, ROW_OP_COLUMN_TYPE), - ]; - /// `epoch`, `seq_id` - pub const PK_TYPES: [DataType; 2] = [EPOCH_COLUMN_TYPE, SEQ_ID_COLUMN_TYPE]; + pub const VNODE_COLUMN_NAME: &str = "kv_log_store_vnode"; pub const EPOCH_COLUMN_TYPE: DataType = DataType::Int64; pub const SEQ_ID_COLUMN_TYPE: DataType = DataType::Int32; pub const ROW_OP_COLUMN_TYPE: DataType = DataType::Int16; + pub const VNODE_COLUMN_TYPE: DataType = DataType::Int16; + + pub mod v1 { + use std::sync::LazyLock; + + use super::*; - pub const EPOCH_COLUMN_INDEX: usize = 0; - pub const SEQ_ID_COLUMN_INDEX: usize = 1; - pub const ROW_OP_COLUMN_INDEX: usize = 2; + /// `epoch`, `seq_id`, `row_op` + pub const KV_LOG_STORE_PREDEFINED_COLUMNS: [(&str, DataType); 3] = [ + (EPOCH_COLUMN_NAME, EPOCH_COLUMN_TYPE), + (SEQ_ID_COLUMN_NAME, SEQ_ID_COLUMN_TYPE), + (ROW_OP_COLUMN_NAME, ROW_OP_COLUMN_TYPE), + ]; + + pub const EPOCH_COLUMN_INDEX: usize = 0; + pub const SEQ_ID_COLUMN_INDEX: usize = 1; + pub const ROW_OP_COLUMN_INDEX: usize = 2; + + /// `epoch`, `seq_id` + pub static PK_ORDERING: LazyLock<[OrderType; 2]> = + LazyLock::new(|| [OrderType::ascending(), OrderType::ascending_nulls_last()]); + } + + pub mod v2 { + use std::sync::LazyLock; + + use super::*; + + /// `epoch`, `seq_id`, `vnode`, `row_op` + pub const KV_LOG_STORE_PREDEFINED_COLUMNS: [(&str, DataType); 4] = [ + (EPOCH_COLUMN_NAME, EPOCH_COLUMN_TYPE), + (SEQ_ID_COLUMN_NAME, SEQ_ID_COLUMN_TYPE), + (VNODE_COLUMN_NAME, VNODE_COLUMN_TYPE), + (ROW_OP_COLUMN_NAME, ROW_OP_COLUMN_TYPE), + ]; + + pub const EPOCH_COLUMN_INDEX: usize = 0; + pub const SEQ_ID_COLUMN_INDEX: usize = 1; + pub const VNODE_COLUMN_INDEX: usize = 2; + pub const ROW_OP_COLUMN_INDEX: usize = 3; + + pub static PK_ORDERING: LazyLock<[OrderType; 3]> = LazyLock::new(|| { + [ + OrderType::ascending(), + OrderType::ascending_nulls_last(), + OrderType::ascending(), + ] + }); + } } diff --git a/src/common/src/error.rs b/src/common/src/error.rs index 791dd786633db..80f46fa7a9e92 100644 --- a/src/common/src/error.rs +++ b/src/common/src/error.rs @@ -13,21 +13,11 @@ // limitations under the License. use std::collections::HashSet; -use std::convert::Infallible; use std::fmt::{Debug, Display, Formatter}; -use std::io::Error as IoError; use std::time::{Duration, SystemTime}; -use memcomparable::Error as MemComparableError; -use risingwave_error::tonic::{ToTonicStatus, TonicStatusWrapper}; -use risingwave_pb::PbFieldNotFound; use thiserror::Error; -use thiserror_ext::{Box, Macro}; -use tokio::task::JoinError; - -use crate::array::ArrayError; -use crate::session_config::SessionConfigError; -use crate::util::value_encoding::error::ValueEncodingError; +use thiserror_ext::Macro; /// Re-export `risingwave_error` for easy access. pub mod v2 { @@ -39,6 +29,7 @@ const ERROR_SUPPRESSOR_RESET_DURATION: Duration = Duration::from_millis(60 * 60 pub trait Error = std::error::Error + Send + Sync + 'static; pub type BoxedError = Box; +#[doc(hidden)] // Used by macros only. pub use anyhow::anyhow as anyhow_error; #[derive(Debug, Clone, Copy, Default)] @@ -102,222 +93,6 @@ impl Display for NoFunction { } } -#[derive(Error, Debug, Box)] -#[thiserror_ext(newtype(name = RwError, backtrace, report_debug))] -pub enum ErrorCode { - #[error("internal error: {0}")] - InternalError(String), - // TODO: unify with the above - #[error(transparent)] - InternalErrorAnyhow( - #[from] - #[backtrace] - anyhow::Error, - ), - #[error("connector error: {0}")] - ConnectorError( - #[source] - #[backtrace] - BoxedError, - ), - #[error(transparent)] - NotImplemented(#[from] NotImplemented), - // Tips: Use this only if it's intended to reject the query - #[error("Not supported: {0}\nHINT: {1}")] - NotSupported(String, String), - #[error(transparent)] - NoFunction(#[from] NoFunction), - #[error(transparent)] - IoError(#[from] IoError), - #[error("Storage error: {0}")] - StorageError( - #[backtrace] - #[source] - BoxedError, - ), - #[error("Expr error: {0}")] - ExprError( - #[source] - #[backtrace] - BoxedError, - ), - // TODO(error-handling): there's a limitation that `#[transparent]` can't be used with `#[backtrace]` if no `#[from]` - // So we emulate a transparent error with "{0}" display here. - #[error("{0}")] - BatchError( - #[source] - #[backtrace] - // `BatchError` - BoxedError, - ), - #[error("Array error: {0}")] - ArrayError( - #[from] - #[backtrace] - ArrayError, - ), - #[error("Stream error: {0}")] - StreamError( - #[backtrace] - #[source] - BoxedError, - ), - // TODO(error-handling): there's a limitation that `#[transparent]` can't be used with `#[backtrace]` if no `#[from]` - // So we emulate a transparent error with "{0}" display here. - #[error("{0}")] - RpcError( - #[source] - #[backtrace] - // `tonic::transport::Error`, `TonicStatusWrapper`, or `RpcError` - BoxedError, - ), - // TODO: use a new type for bind error - // TODO(error-handling): should prefer use error types than strings. - #[error("Bind error: {0}")] - BindError(String), - // TODO: only keep this one - #[error("Failed to bind expression: {expr}: {error}")] - BindErrorRoot { - expr: String, - #[source] - #[backtrace] - error: BoxedError, - }, - #[error("Catalog error: {0}")] - CatalogError( - #[source] - #[backtrace] - BoxedError, - ), - #[error("Protocol error: {0}")] - ProtocolError(String), - #[error("Scheduler error: {0}")] - SchedulerError( - #[source] - #[backtrace] - BoxedError, - ), - #[error("Task not found")] - TaskNotFound, - #[error("Session not found")] - SessionNotFound, - #[error("Item not found: {0}")] - ItemNotFound(String), - #[error("Invalid input syntax: {0}")] - InvalidInputSyntax(String), - #[error("Can not compare in memory: {0}")] - MemComparableError(#[from] MemComparableError), - #[error("Error while de/se values: {0}")] - ValueEncodingError( - #[from] - #[backtrace] - ValueEncodingError, - ), - #[error("Invalid value `{config_value}` for `{config_entry}`")] - InvalidConfigValue { - config_entry: String, - config_value: String, - }, - #[error("Invalid Parameter Value: {0}")] - InvalidParameterValue(String), - #[error("Sink error: {0}")] - SinkError( - #[source] - #[backtrace] - BoxedError, - ), - #[error("Permission denied: {0}")] - PermissionDenied(String), - #[error("Failed to get/set session config: {0}")] - SessionConfig( - #[from] - #[backtrace] - SessionConfigError, - ), -} - -impl From for tonic::Status { - fn from(err: RwError) -> Self { - use tonic::Code; - - let code = match err.inner() { - ErrorCode::ExprError(_) => Code::InvalidArgument, - ErrorCode::PermissionDenied(_) => Code::PermissionDenied, - ErrorCode::InternalError(_) => Code::Internal, - _ => Code::Internal, - }; - - err.to_status_unnamed(code) - } -} - -impl From for RwError { - fn from(status: TonicStatusWrapper) -> Self { - use tonic::Code; - - let message = status.inner().message(); - - // TODO(error-handling): `message` loses the source chain. - match status.inner().code() { - Code::InvalidArgument => ErrorCode::InvalidParameterValue(message.to_string()), - Code::NotFound | Code::AlreadyExists => ErrorCode::CatalogError(status.into()), - Code::PermissionDenied => ErrorCode::PermissionDenied(message.to_string()), - Code::Cancelled => ErrorCode::SchedulerError(status.into()), - _ => ErrorCode::RpcError(status.into()), - } - .into() - } -} - -impl From for RwError { - fn from(status: tonic::Status) -> Self { - // Always wrap the status. - Self::from(TonicStatusWrapper::new(status)) - } -} - -impl From for RwError { - fn from(join_error: JoinError) -> Self { - anyhow::anyhow!(join_error).into() - } -} - -impl From for RwError { - fn from(addr_parse_error: std::net::AddrParseError) -> Self { - anyhow::anyhow!(addr_parse_error).into() - } -} - -impl From for RwError { - fn from(x: Infallible) -> Self { - match x {} - } -} - -impl From for RwError { - fn from(e: String) -> Self { - ErrorCode::InternalError(e).into() - } -} - -impl From for RwError { - fn from(err: PbFieldNotFound) -> Self { - ErrorCode::InternalError(format!( - "Failed to decode prost: field not found `{}`", - err.0 - )) - .into() - } -} - -impl From for RwError { - fn from(err: tonic::transport::Error) -> Self { - ErrorCode::RpcError(err.into()).into() - } -} - -pub type Result = std::result::Result; - /// Util macro for generating error when condition check failed. /// /// # Case 1: Expression only. @@ -456,13 +231,10 @@ mod tests { use anyhow::anyhow; use super::*; - use crate::error::ErrorCode::InternalErrorAnyhow; - #[test] - fn test_display_internal_error() { - let internal_error = ErrorCode::InternalError("some thing bad happened!".to_string()); - println!("{:?}", RwError::from(internal_error)); - } + #[derive(Error, Debug)] + #[error(transparent)] + struct MyError(#[from] anyhow::Error); #[test] fn test_ensure() { @@ -472,41 +244,31 @@ mod tests { let err_msg = "a < 0"; let error = (|| { ensure!(a < 0); - Ok::<_, RwError>(()) + Ok::<_, MyError>(()) })() .unwrap_err(); - assert_eq!( - RwError::from(InternalErrorAnyhow(anyhow!(err_msg))).to_string(), - error.to_string(), - ); + assert_eq!(MyError(anyhow!(err_msg)).to_string(), error.to_string(),); } { let err_msg = "error msg without args"; let error = (|| { ensure!(a < 0, "error msg without args"); - Ok::<_, RwError>(()) + Ok::<_, MyError>(()) })() .unwrap_err(); - assert_eq!( - RwError::from(InternalErrorAnyhow(anyhow!(err_msg))).to_string(), - error.to_string() - ); + assert_eq!(MyError(anyhow!(err_msg)).to_string(), error.to_string()); } { let error = (|| { ensure!(a < 0, "error msg with args: {}", "xx"); - Ok::<_, RwError>(()) + Ok::<_, MyError>(()) })() .unwrap_err(); assert_eq!( - RwError::from(InternalErrorAnyhow(anyhow!( - "error msg with args: {}", - "xx" - ))) - .to_string(), + MyError(anyhow!("error msg with args: {}", "xx")).to_string(), error.to_string() ); } @@ -514,7 +276,7 @@ mod tests { #[test] fn test_ensure_eq() { - fn ensure_a_equals_b() -> Result<()> { + fn ensure_a_equals_b() -> Result<(), MyError> { let a = 1; let b = 2; ensure_eq!(a, b); @@ -523,31 +285,4 @@ mod tests { let err = ensure_a_equals_b().unwrap_err(); assert_eq!(err.to_string(), "a == b assertion failed (a is 1, b is 2)"); } - - #[test] - fn test_into() { - use tonic::{Code, Status}; - fn check_grpc_error(ec: ErrorCode, grpc_code: Code) { - assert_eq!(Status::from(RwError::from(ec)).code(), grpc_code); - } - - check_grpc_error(ErrorCode::TaskNotFound, Code::Internal); - check_grpc_error(ErrorCode::InternalError(String::new()), Code::Internal); - check_grpc_error( - ErrorCode::NotImplemented(not_implemented!("test")), - Code::Internal, - ); - } - - #[test] - #[ignore] // it's not a good practice to include error source in `Display`, see #13248 - fn test_internal_sources() { - use anyhow::Context; - - let res: Result<()> = Err(anyhow::anyhow!("inner")) - .context("outer") - .map_err(Into::into); - - assert_eq!(res.unwrap_err().to_string(), "internal error: outer: inner"); - } } diff --git a/src/common/src/estimate_size/collections/lru.rs b/src/common/src/estimate_size/collections/lru.rs index 523ca390837c3..0e7ec7b3862b2 100644 --- a/src/common/src/estimate_size/collections/lru.rs +++ b/src/common/src/estimate_size/collections/lru.rs @@ -16,7 +16,7 @@ use std::alloc::{Allocator, Global}; use std::borrow::Borrow; use std::hash::{BuildHasher, Hash}; -use lru::{DefaultHasher, KeyRef, LruCache}; +use lru::{DefaultHasher, LruCache}; use super::{AtomicMutGuard, MutGuard}; use crate::estimate_size::{EstimateSize, KvSize}; @@ -66,7 +66,7 @@ impl(&mut self, k: &Q) -> Option<&V> where - KeyRef: Borrow, + K: Borrow, Q: Hash + Eq + ?Sized, { self.inner.get(k) @@ -100,7 +100,7 @@ impl(&self, k: &Q) -> bool where - KeyRef: Borrow, + K: Borrow, Q: Hash + Eq + ?Sized, { self.inner.contains(k) diff --git a/src/common/src/field_generator/mod.rs b/src/common/src/field_generator/mod.rs index 0309798068854..679d60ba1f188 100644 --- a/src/common/src/field_generator/mod.rs +++ b/src/common/src/field_generator/mod.rs @@ -18,6 +18,7 @@ mod varchar; use std::time::Duration; +// TODO(error-handling): use a new error type use anyhow::{anyhow, Result}; use chrono::{DateTime, FixedOffset}; pub use numeric::*; diff --git a/src/common/src/hash/key.rs b/src/common/src/hash/key.rs index 2023636464f92..4e7072a4abdf4 100644 --- a/src/common/src/hash/key.rs +++ b/src/common/src/hash/key.rs @@ -698,8 +698,7 @@ mod tests { 100, PrecomputedBuildHasher, ); - let keys = - K::build(column_indexes.as_slice(), &data).expect("Failed to build hash keys"); + let keys = K::build_many(column_indexes.as_slice(), &data); for (row_id, key) in keys.into_iter().enumerate() { let row_ids = fast_hash_map.entry(key).or_default(); @@ -741,7 +740,7 @@ mod tests { F: FnOnce() -> (DataChunk, Vec), { let (data, types) = data_gen(); - let keys = K::build(column_indexes.as_slice(), &data).expect("Failed to build hash keys"); + let keys = K::build_many(column_indexes.as_slice(), &data); let mut array_builders = column_indexes .iter() @@ -839,15 +838,14 @@ mod tests { // losslessly. #[test] fn test_simple_hash_key_nullable_serde() { - let keys = Key64::build( + let keys = Key64::build_many( &[0, 1], &DataChunk::from_pretty( "i i 1 . . 2", ), - ) - .unwrap(); + ); let mut array_builders = [0, 1] .iter() diff --git a/src/common/src/hash/key_v2.rs b/src/common/src/hash/key_v2.rs index 4644e0191de61..227944d07b3bc 100644 --- a/src/common/src/hash/key_v2.rs +++ b/src/common/src/hash/key_v2.rs @@ -224,9 +224,8 @@ pub trait HashKey: // TODO: rename to `NullBitmap` and note that high bit represents null! type Bitmap: NullBitmap; - // TODO: remove result and rename to `build_many` /// Build hash keys from the given `data_chunk` with `column_indices` in a batch. - fn build(column_indices: &[usize], data_chunk: &DataChunk) -> ArrayResult>; + fn build_many(column_indices: &[usize], data_chunk: &DataChunk) -> Vec; /// Deserializes the hash key into a row. fn deserialize(&self, data_types: &[DataType]) -> ArrayResult; @@ -288,7 +287,7 @@ impl EstimateSize for HashKeyImpl { impl HashKey for HashKeyImpl { type Bitmap = N; - fn build(column_indices: &[usize], data_chunk: &DataChunk) -> ArrayResult> { + fn build_many(column_indices: &[usize], data_chunk: &DataChunk) -> Vec { let hash_codes = data_chunk.get_hash_values(column_indices, XxHash64Builder); let mut serializers = { @@ -331,8 +330,7 @@ impl HashKey for HashKeyImpl { }); } - let hash_keys = serializers.into_iter().map(|s| s.finish()).collect(); - Ok(hash_keys) + serializers.into_iter().map(|s| s.finish()).collect() } fn deserialize(&self, data_types: &[DataType]) -> ArrayResult { diff --git a/src/common/src/hash/mod.rs b/src/common/src/hash/mod.rs index c6769d7cb4266..8f2c89cec654d 100644 --- a/src/common/src/hash/mod.rs +++ b/src/common/src/hash/mod.rs @@ -16,6 +16,7 @@ mod consistent_hash; // TODO: move this to a separate module mod dispatcher; mod key; mod key_v2; +pub mod table_distribution; pub use consistent_hash::bitmap::*; pub use consistent_hash::mapping::*; diff --git a/src/common/src/hash/table_distribution.rs b/src/common/src/hash/table_distribution.rs new file mode 100644 index 0000000000000..c5bb2fd3d8dd2 --- /dev/null +++ b/src/common/src/hash/table_distribution.rs @@ -0,0 +1,283 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::mem::replace; +use std::ops::Deref; +use std::sync::{Arc, LazyLock}; + +use itertools::Itertools; +use risingwave_pb::plan_common::StorageTableDesc; +use tracing::warn; + +use crate::array::{Array, DataChunk, PrimitiveArray}; +use crate::buffer::{Bitmap, BitmapBuilder}; +use crate::hash::VirtualNode; +use crate::row::Row; +use crate::util::iter_util::ZipEqFast; + +/// For tables without distribution (singleton), the `DEFAULT_VNODE` is encoded. +pub const DEFAULT_VNODE: VirtualNode = VirtualNode::ZERO; + +#[derive(Debug, Clone)] +enum ComputeVnode { + Singleton, + DistKeyIndices { + /// Indices of distribution key for computing vnode, based on the pk columns of the table. + dist_key_in_pk_indices: Vec, + }, + VnodeColumnIndex { + /// Indices of vnode columns. + vnode_col_idx_in_pk: usize, + }, +} + +#[derive(Debug, Clone)] +/// Represents the distribution for a specific table instance. +pub struct TableDistribution { + /// The way to compute vnode provided primary key + compute_vnode: ComputeVnode, + + /// Virtual nodes that the table is partitioned into. + vnodes: Arc, +} + +pub const SINGLETON_VNODE: VirtualNode = DEFAULT_VNODE; + +impl TableDistribution { + pub fn new_from_storage_table_desc( + vnodes: Option>, + table_desc: &StorageTableDesc, + ) -> Self { + let dist_key_in_pk_indices = table_desc + .dist_key_in_pk_indices + .iter() + .map(|&k| k as usize) + .collect_vec(); + let vnode_col_idx_in_pk = table_desc.vnode_col_idx_in_pk.map(|k| k as usize); + Self::new(vnodes, dist_key_in_pk_indices, vnode_col_idx_in_pk) + } + + pub fn new( + vnodes: Option>, + dist_key_in_pk_indices: Vec, + vnode_col_idx_in_pk: Option, + ) -> Self { + let compute_vnode = if let Some(vnode_col_idx_in_pk) = vnode_col_idx_in_pk { + ComputeVnode::VnodeColumnIndex { + vnode_col_idx_in_pk, + } + } else if !dist_key_in_pk_indices.is_empty() { + ComputeVnode::DistKeyIndices { + dist_key_in_pk_indices, + } + } else { + ComputeVnode::Singleton + }; + + let vnodes = vnodes.unwrap_or_else(Self::singleton_vnode_bitmap); + if let ComputeVnode::Singleton = &compute_vnode { + if &vnodes != Self::singleton_vnode_bitmap_ref() { + warn!( + ?vnodes, + "singleton distribution get non-singleton vnode bitmap" + ); + } + } + + Self { + compute_vnode, + vnodes, + } + } + + pub fn is_singleton(&self) -> bool { + matches!(&self.compute_vnode, ComputeVnode::Singleton) + } + + pub fn singleton_vnode_bitmap_ref() -> &'static Arc { + /// A bitmap that only the default vnode is set. + static SINGLETON_VNODES: LazyLock> = LazyLock::new(|| { + let mut vnodes = BitmapBuilder::zeroed(VirtualNode::COUNT); + vnodes.set(SINGLETON_VNODE.to_index(), true); + vnodes.finish().into() + }); + + SINGLETON_VNODES.deref() + } + + pub fn singleton_vnode_bitmap() -> Arc { + Self::singleton_vnode_bitmap_ref().clone() + } + + pub fn all_vnodes_ref() -> &'static Arc { + /// A bitmap that all vnodes are set. + static ALL_VNODES: LazyLock> = + LazyLock::new(|| Bitmap::ones(VirtualNode::COUNT).into()); + &ALL_VNODES + } + + pub fn all_vnodes() -> Arc { + Self::all_vnodes_ref().clone() + } + + /// Distribution that accesses all vnodes, mainly used for tests. + pub fn all(dist_key_in_pk_indices: Vec) -> Self { + Self { + compute_vnode: ComputeVnode::DistKeyIndices { + dist_key_in_pk_indices, + }, + vnodes: Self::all_vnodes(), + } + } + + /// Fallback distribution for singleton or tests. + pub fn singleton() -> Self { + Self { + compute_vnode: ComputeVnode::Singleton, + vnodes: Self::singleton_vnode_bitmap(), + } + } + + pub fn update_vnode_bitmap(&mut self, new_vnodes: Arc) -> Arc { + if self.is_singleton() && &new_vnodes != Self::singleton_vnode_bitmap_ref() { + warn!(?new_vnodes, "update vnode on singleton distribution"); + } + assert_eq!(self.vnodes.len(), new_vnodes.len()); + replace(&mut self.vnodes, new_vnodes) + } + + pub fn vnodes(&self) -> &Arc { + &self.vnodes + } + + /// Get vnode value with given primary key. + pub fn compute_vnode_by_pk(&self, pk: impl Row) -> VirtualNode { + match &self.compute_vnode { + ComputeVnode::Singleton => SINGLETON_VNODE, + ComputeVnode::DistKeyIndices { + dist_key_in_pk_indices, + } => compute_vnode(pk, dist_key_in_pk_indices, &self.vnodes), + ComputeVnode::VnodeColumnIndex { + vnode_col_idx_in_pk, + } => get_vnode_from_row(pk, *vnode_col_idx_in_pk, &self.vnodes), + } + } + + pub fn try_compute_vnode_by_pk_prefix(&self, pk_prefix: impl Row) -> Option { + match &self.compute_vnode { + ComputeVnode::Singleton => Some(SINGLETON_VNODE), + ComputeVnode::DistKeyIndices { + dist_key_in_pk_indices, + } => dist_key_in_pk_indices + .iter() + .all(|&d| d < pk_prefix.len()) + .then(|| compute_vnode(pk_prefix, dist_key_in_pk_indices, &self.vnodes)), + ComputeVnode::VnodeColumnIndex { + vnode_col_idx_in_pk, + } => { + if *vnode_col_idx_in_pk >= pk_prefix.len() { + None + } else { + Some(get_vnode_from_row( + pk_prefix, + *vnode_col_idx_in_pk, + &self.vnodes, + )) + } + } + } + } +} + +/// Get vnode value with `indices` on the given `row`. +pub fn compute_vnode(row: impl Row, indices: &[usize], vnodes: &Bitmap) -> VirtualNode { + assert!(!indices.is_empty()); + let vnode = VirtualNode::compute_row(&row, indices); + check_vnode_is_set(vnode, vnodes); + + tracing::debug!(target: "events::storage::storage_table", "compute vnode: {:?} key {:?} => {}", row, indices, vnode); + + vnode +} + +pub fn get_vnode_from_row(row: impl Row, index: usize, vnodes: &Bitmap) -> VirtualNode { + let vnode = VirtualNode::from_datum(row.datum_at(index)); + check_vnode_is_set(vnode, vnodes); + + tracing::debug!(target: "events::storage::storage_table", "get vnode from row: {:?} vnode column index {:?} => {}", row, index, vnode); + + vnode +} + +impl TableDistribution { + /// Get vnode values with `indices` on the given `chunk`. + /// + /// Vnode of invisible rows will be included. Only the vnode of visible row check if it's accessible + pub fn compute_chunk_vnode(&self, chunk: &DataChunk, pk_indices: &[usize]) -> Vec { + match &self.compute_vnode { + ComputeVnode::Singleton => { + vec![SINGLETON_VNODE; chunk.capacity()] + } + ComputeVnode::DistKeyIndices { + dist_key_in_pk_indices, + } => { + let dist_key_indices = dist_key_in_pk_indices + .iter() + .map(|idx| pk_indices[*idx]) + .collect_vec(); + + VirtualNode::compute_chunk(chunk, &dist_key_indices) + .into_iter() + .zip_eq_fast(chunk.visibility().iter()) + .map(|(vnode, vis)| { + // Ignore the invisible rows. + if vis { + check_vnode_is_set(vnode, &self.vnodes); + } + vnode + }) + .collect() + } + ComputeVnode::VnodeColumnIndex { + vnode_col_idx_in_pk, + } => { + let array: &PrimitiveArray = + chunk.columns()[pk_indices[*vnode_col_idx_in_pk]].as_int16(); + array + .raw_iter() + .zip_eq_fast(array.null_bitmap().iter()) + .zip_eq_fast(chunk.visibility().iter()) + .map(|((vnode, exist), vis)| { + let vnode = VirtualNode::from_scalar(vnode); + if vis { + assert!(exist); + check_vnode_is_set(vnode, &self.vnodes); + } + vnode + }) + .collect_vec() + } + } + } +} + +/// Check whether the given `vnode` is set in the `vnodes` of this table. +fn check_vnode_is_set(vnode: VirtualNode, vnodes: &Bitmap) { + let is_set = vnodes.is_set(vnode.to_index()); + assert!( + is_set, + "vnode {} should not be accessed by this table", + vnode + ); +} diff --git a/src/common/src/lib.rs b/src/common/src/lib.rs index 20428599b1039..313c0bada6616 100644 --- a/src/common/src/lib.rs +++ b/src/common/src/lib.rs @@ -80,7 +80,6 @@ pub mod test_utils; pub mod transaction; pub mod types; pub mod vnode_mapping; - pub mod test_prelude { pub use super::array::{DataChunkTestExt, StreamChunkTestExt}; pub use super::catalog::test_utils::ColumnDescTestExt; @@ -93,9 +92,9 @@ pub const UNKNOWN_GIT_SHA: &str = "unknown"; // The single source of truth of the pg parameters, Used in ConfigMap and current_cluster_version. // The version of PostgreSQL that Risingwave claims to be. -pub const PG_VERSION: &str = "9.5.0"; +pub const PG_VERSION: &str = "13.14.0"; /// The version of PostgreSQL that Risingwave claims to be. -pub const SERVER_VERSION_NUM: i32 = 90500; +pub const SERVER_VERSION_NUM: i32 = 130014; /// Shows the server-side character set encoding. At present, this parameter can be shown but not set, because the encoding is determined at database creation time. It is also the default value of `client_encoding`. pub const SERVER_ENCODING: &str = "UTF8"; /// see diff --git a/src/common/src/log.rs b/src/common/src/log.rs index f462c6c5e13da..b320a111f4915 100644 --- a/src/common/src/log.rs +++ b/src/common/src/log.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::num::NonZeroU32; +use std::num::{NonZeroU32, NonZeroUsize}; use std::sync::atomic::{AtomicUsize, Ordering}; use governor::Quota; @@ -46,10 +46,13 @@ impl LogSuppresser { /// Check if the log should be suppressed. /// If the log should be suppressed, return `Err(LogSuppressed)`. - /// Otherwise, return `Ok(usize)` with count of suppressed messages before. - pub fn check(&self) -> core::result::Result { + /// Otherwise, return `Ok(Some(..))` with count of suppressed messages since last check, + /// or `Ok(None)` if there's none. + pub fn check(&self) -> core::result::Result, LogSuppressed> { match self.rate_limiter.check() { - Ok(()) => Ok(self.suppressed_count.swap(0, Ordering::Relaxed)), + Ok(()) => Ok(NonZeroUsize::new( + self.suppressed_count.swap(0, Ordering::Relaxed), + )), Err(_) => { self.suppressed_count.fetch_add(1, Ordering::Relaxed); Err(LogSuppressed) @@ -72,10 +75,16 @@ mod tests { use std::sync::LazyLock; use std::time::Duration; + use tracing_subscriber::util::SubscriberInitExt; + use super::*; #[tokio::test] async fn demo() { + let _logger = tracing_subscriber::fmt::Subscriber::builder() + .with_max_level(tracing::Level::ERROR) + .set_default(); + let mut interval = tokio::time::interval(Duration::from_millis(100)); for _ in 0..100 { interval.tick().await; @@ -86,7 +95,7 @@ mod tests { }); if let Ok(suppressed_count) = RATE_LIMITER.check() { - println!("failed to foo bar. suppressed_count = {}", suppressed_count); + tracing::error!(suppressed_count, "failed to foo bar"); } } } diff --git a/src/common/src/metrics/error_metrics.rs b/src/common/src/metrics/error_metrics.rs index 8cb1d8082de4b..daf9f293bf09f 100644 --- a/src/common/src/metrics/error_metrics.rs +++ b/src/common/src/metrics/error_metrics.rs @@ -86,6 +86,7 @@ pub struct ErrorMetrics { pub user_compute_error: ErrorMetricRef<4>, pub user_source_reader_error: ErrorMetricRef<5>, pub user_source_error: ErrorMetricRef<5>, + pub cdc_source_error: ErrorMetricRef<3>, } impl ErrorMetrics { @@ -123,6 +124,12 @@ impl ErrorMetrics { "table_id", ], )), + // cdc source is singleton, so we use source_id to identify the connector + cdc_source_error: Arc::new(ErrorMetric::new( + "cdc_source_error", + "CDC source errors in the system, queryable by tags", + &["connector_name", "source_id", "error_msg"], + )), } } @@ -132,6 +139,7 @@ impl ErrorMetrics { &self.user_compute_error.desc, &self.user_source_reader_error.desc, &self.user_source_error.desc, + &self.cdc_source_error.desc, ] } @@ -141,6 +149,7 @@ impl ErrorMetrics { self.user_compute_error.collect(), self.user_source_reader_error.collect(), self.user_source_error.collect(), + self.cdc_source_error.collect(), ] } } diff --git a/src/common/src/metrics/guarded_metrics.rs b/src/common/src/metrics/guarded_metrics.rs index bda396128445c..f1f8bdc9c8765 100644 --- a/src/common/src/metrics/guarded_metrics.rs +++ b/src/common/src/metrics/guarded_metrics.rs @@ -160,6 +160,20 @@ impl LabelGuardedMetricsInfo { } } +/// `LabelGuardedMetricVec` enhances the [`MetricVec`] to ensure the set of labels to be +/// correctly removed from the Prometheus client once being dropped. This is useful for metrics +/// that are associated with an object that can be dropped, such as streaming jobs, fragments, +/// actors, batch tasks, etc. +/// +/// When a set labels is dropped, it will record it in the `uncollected_removed_labels` set. +/// Once the metrics has been collected, it will finally remove the metrics of the labels. +/// +/// See also [`LabelGuardedMetricsInfo`] and [`LabelGuard::drop`]. +/// +/// # Arguments +/// +/// * `T` - The type of the raw metrics vec. +/// * `N` - The number of labels. #[derive(Clone)] pub struct LabelGuardedMetricVec { inner: MetricVec, diff --git a/src/common/src/row/mod.rs b/src/common/src/row/mod.rs index 924e3db490a75..8114b96cb37e5 100644 --- a/src/common/src/row/mod.rs +++ b/src/common/src/row/mod.rs @@ -322,6 +322,10 @@ impl Row for [D; N] { impl_slice_row!(); } +impl Row for ArrayVec<[D; N]> { + impl_slice_row!(); +} + /// Implements [`Row`] for an optional row. impl Row for Option { fn datum_at(&self, index: usize) -> DatumRef<'_> { @@ -451,6 +455,7 @@ mod owned_row; mod project; mod repeat_n; mod slice; +pub use ::tinyvec::ArrayVec; pub use chain::Chain; pub use compacted_row::CompactedRow; pub use empty::{empty, Empty}; diff --git a/src/common/src/session_config/mod.rs b/src/common/src/session_config/mod.rs index daef5faf1e240..83e8d85429605 100644 --- a/src/common/src/session_config/mod.rs +++ b/src/common/src/session_config/mod.rs @@ -94,6 +94,11 @@ pub struct ConfigMap { #[parameter(default = true, rename = "rw_batch_enable_sort_agg")] batch_enable_sort_agg: bool, + /// Enable distributed DML, so an insert, delete, and update statement can be executed in a distributed way (e.g. running in multiple compute nodes). + /// No atomicity guarantee in this mode. Its goal is to gain the best ingestion performance for initial batch ingestion where users always can drop their table when failure happens. + #[parameter(default = false, rename = "batch_enable_distributed_dml")] + batch_enable_distributed_dml: bool, + /// The max gap allowed to transform small range scan into multi point lookup. #[parameter(default = 8)] max_split_range_gap: i32, @@ -207,6 +212,10 @@ pub struct ConfigMap { #[parameter(default = 0u32)] statement_timeout: u32, + /// Terminate any session that has been idle (that is, waiting for a client query) within an open transaction for longer than the specified amount of time in milliseconds. + #[parameter(default = 60000u32)] + idle_in_transaction_session_timeout: u32, + /// See /// Unused in RisingWave, support for compatibility. #[parameter(default = 0)] diff --git a/src/common/src/session_config/sink_decouple.rs b/src/common/src/session_config/sink_decouple.rs index fbb7f684ccabf..19f2a8ec81561 100644 --- a/src/common/src/session_config/sink_decouple.rs +++ b/src/common/src/session_config/sink_decouple.rs @@ -38,12 +38,16 @@ impl FromStr for SinkDecouple { } } -impl ToString for SinkDecouple { - fn to_string(&self) -> String { - match self { - Self::Default => "default".to_string(), - Self::Enable => "enable".to_string(), - Self::Disable => "disable".to_string(), - } +impl std::fmt::Display for SinkDecouple { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "{}", + match self { + Self::Default => "default", + Self::Enable => "enable", + Self::Disable => "disable", + } + ) } } diff --git a/src/common/src/system_param/common.rs b/src/common/src/system_param/common.rs index b8dcf825e2dda..d8ff741399533 100644 --- a/src/common/src/system_param/common.rs +++ b/src/common/src/system_param/common.rs @@ -12,37 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Mutex; - +use super::diff::SystemParamsDiff; use super::reader::SystemParamsReader; use crate::util::tracing::layer::toggle_otel_layer; /// Node-independent handler for system parameter changes. /// /// Currently, it is only used to enable or disable the distributed tracing layer. -pub struct CommonHandler { - last_params: Mutex>, -} +#[derive(Debug)] +pub struct CommonHandler; impl CommonHandler { /// Create a new handler with the initial parameters. pub fn new(initial: SystemParamsReader) -> Self { - let this = Self { - last_params: None.into(), - }; - this.handle_change(initial); + let this = Self; + this.handle_change(&SystemParamsDiff::from_initial(initial)); this } /// Handle the change of system parameters. - // TODO: directly call this method with the difference of old and new params. - pub fn handle_change(&self, new_params: SystemParamsReader) { - let mut last_params = self.last_params.lock().unwrap(); - - if last_params.as_ref().map(|p| p.enable_tracing()) != Some(new_params.enable_tracing()) { - toggle_otel_layer(new_params.enable_tracing()); + pub fn handle_change(&self, diff: &SystemParamsDiff) { + if let Some(enabled) = diff.enable_tracing { + toggle_otel_layer(enabled) } - - last_params.replace(new_params); } } diff --git a/src/common/src/system_param/diff.rs b/src/common/src/system_param/diff.rs new file mode 100644 index 0000000000000..243b3e247bec8 --- /dev/null +++ b/src/common/src/system_param/diff.rs @@ -0,0 +1,67 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use super::reader::SystemParamsRead; +use crate::for_all_params; + +macro_rules! define_diff { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr, $doc:literal, $($rest:tt)* },)*) => { + /// The diff of the system params. + /// + /// Fields that are changed are set to `Some`, otherwise `None`. + #[derive(Default, Debug, Clone)] + pub struct SystemParamsDiff { + $( + #[doc = $doc] + pub $field: Option<$type>, + )* + } + } +} +for_all_params!(define_diff); + +impl SystemParamsDiff { + /// Create a diff between the given two system params. + pub fn diff(prev: impl SystemParamsRead, curr: impl SystemParamsRead) -> Self { + let mut diff = Self::default(); + + macro_rules! set_diff_field { + ($({ $field:ident, $($rest:tt)* },)*) => { + $( + if curr.$field() != prev.$field() { + diff.$field = Some(curr.$field().to_owned()); + } + )* + }; + } + for_all_params!(set_diff_field); + + diff + } + + /// Create a diff from the given initial system params. + /// All fields will be set to `Some`. + pub fn from_initial(initial: impl SystemParamsRead) -> Self { + macro_rules! initial_field { + ($({ $field:ident, $($rest:tt)* },)*) => { + Self { + $( + $field: Some(initial.$field().to_owned()), + )* + } + }; + } + for_all_params!(initial_field) + } +} diff --git a/src/common/src/system_param/local_manager.rs b/src/common/src/system_param/local_manager.rs index 312c5577a0f81..5040d30f811d0 100644 --- a/src/common/src/system_param/local_manager.rs +++ b/src/common/src/system_param/local_manager.rs @@ -20,6 +20,7 @@ use risingwave_pb::meta::SystemParams; use tokio::sync::watch::{channel, Receiver, Sender}; use super::common::CommonHandler; +use super::diff::SystemParamsDiff; use super::reader::SystemParamsReader; use super::system_params_for_test; @@ -41,28 +42,39 @@ pub struct LocalSystemParamsManager { } impl LocalSystemParamsManager { + /// Create a new instance of `LocalSystemParamsManager` and spawn a task to run + /// the common handler. pub fn new(initial_params: SystemParamsReader) -> Self { - let params = Arc::new(ArcSwap::from_pointee(initial_params.clone())); - let (tx, _) = channel(params.clone()); + let this = Self::new_inner(initial_params.clone()); // Spawn a task to run the common handler. tokio::spawn({ - let mut rx = tx.subscribe(); + let mut rx = this.tx.subscribe(); async move { + let mut params = initial_params.clone(); let handler = CommonHandler::new(initial_params); while rx.changed().await.is_ok() { + // TODO: directly watch the changes instead of diffing ourselves. let new_params = (**rx.borrow_and_update().load()).clone(); - handler.handle_change(new_params); + let diff = SystemParamsDiff::diff(params.as_ref(), new_params.as_ref()); + handler.handle_change(&diff); + params = new_params; } } }); - Self { params, tx } + this } pub fn for_test() -> Self { - Self::new(system_params_for_test().into()) + Self::new_inner(system_params_for_test().into()) + } + + fn new_inner(initial_params: SystemParamsReader) -> Self { + let params = Arc::new(ArcSwap::from_pointee(initial_params)); + let (tx, _) = channel(params.clone()); + Self { params, tx } } pub fn get_params(&self) -> SystemParamsReaderRef { @@ -89,12 +101,11 @@ mod tests { #[tokio::test] async fn test_manager() { - let p = SystemParams::default().into(); - let manager = LocalSystemParamsManager::new(p); + let manager = LocalSystemParamsManager::for_test(); let shared_params = manager.get_params(); let new_params = SystemParams { - sstable_size_mb: Some(1), + sstable_size_mb: Some(114514), ..Default::default() }; diff --git a/src/common/src/system_param/mod.rs b/src/common/src/system_param/mod.rs index cffa7a4564a5f..19c36baf09c68 100644 --- a/src/common/src/system_param/mod.rs +++ b/src/common/src/system_param/mod.rs @@ -21,22 +21,51 @@ //! - Add a new method to [`reader::SystemParamsReader`]. pub mod common; +pub mod diff; pub mod local_manager; pub mod reader; use std::fmt::Debug; use std::ops::RangeBounds; +use std::str::FromStr; use paste::paste; use risingwave_pb::meta::PbSystemParams; +use self::diff::SystemParamsDiff; + pub type SystemParamsError = String; type Result = core::result::Result; +/// The trait for the value type of a system parameter. +pub trait ParamValue: ToString + FromStr { + type Borrowed<'a>; +} + +macro_rules! impl_param_value { + ($type:ty) => { + impl_param_value!($type => $type); + }; + ($type:ty => $borrowed:ty) => { + impl ParamValue for $type { + type Borrowed<'a> = $borrowed; + } + }; +} + +impl_param_value!(bool); +impl_param_value!(u32); +impl_param_value!(u64); +impl_param_value!(f64); +impl_param_value!(String => &'a str); + /// Define all system parameters here. /// -/// Macro input is { field identifier, type, default value, is mutable } +/// To match all these information, write the match arm as follows: +/// ```text +/// ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr, $doc:literal, $($rest:tt)* },)*) => { +/// ``` /// /// Note: /// - Having `None` as default value means the parameter must be initialized. @@ -44,20 +73,20 @@ type Result = core::result::Result; macro_rules! for_all_params { ($macro:ident) => { $macro! { - { barrier_interval_ms, u32, Some(1000_u32), true }, - { checkpoint_frequency, u64, Some(1_u64), true }, - { sstable_size_mb, u32, Some(256_u32), false }, - { parallel_compact_size_mb, u32, Some(512_u32), false }, - { block_size_kb, u32, Some(64_u32), false }, - { bloom_false_positive, f64, Some(0.001_f64), false }, - { state_store, String, None, false }, - { data_directory, String, None, false }, - { backup_storage_url, String, Some("memory".to_string()), true }, - { backup_storage_directory, String, Some("backup".to_string()), true }, - { max_concurrent_creating_streaming_jobs, u32, Some(1_u32), true }, - { pause_on_next_bootstrap, bool, Some(false), true }, - { wasm_storage_url, String, Some("fs://.risingwave/data".to_string()), false }, - { enable_tracing, bool, Some(false), true }, + // name type default value mut? doc + { barrier_interval_ms, u32, Some(1000_u32), true, "The interval of periodic barrier.", }, + { checkpoint_frequency, u64, Some(1_u64), true, "There will be a checkpoint for every n barriers.", }, + { sstable_size_mb, u32, Some(256_u32), false, "Target size of the Sstable.", }, + { parallel_compact_size_mb, u32, Some(512_u32), false, "", }, + { block_size_kb, u32, Some(64_u32), false, "Size of each block in bytes in SST.", }, + { bloom_false_positive, f64, Some(0.001_f64), false, "False positive probability of bloom filter.", }, + { state_store, String, None, false, "", }, + { data_directory, String, None, false, "Remote directory for storing data and metadata objects.", }, + { backup_storage_url, String, None, true, "Remote storage url for storing snapshots.", }, + { backup_storage_directory, String, None, true, "Remote directory for storing snapshots.", }, + { max_concurrent_creating_streaming_jobs, u32, Some(1_u32), true, "Max number of concurrent creating streaming jobs.", }, + { pause_on_next_bootstrap, bool, Some(false), true, "Whether to pause all data sources on next bootstrap.", }, + { enable_tracing, bool, Some(false), true, "Whether to enable distributed tracing.", }, } }; } @@ -72,7 +101,7 @@ macro_rules! key_of { /// Define key constants for fields in `PbSystemParams` for use of other modules. macro_rules! def_key { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $($rest:tt)* },)*) => { paste! { $( pub const [<$field:upper _KEY>]: &str = key_of!($field); @@ -83,23 +112,47 @@ macro_rules! def_key { for_all_params!(def_key); -/// Define default value functions. -macro_rules! def_default { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { - pub mod default { - $( - pub fn $field() -> Option<$type> { +/// Define default value functions returning `Option`. +macro_rules! def_default_opt { + ($({ $field:ident, $type:ty, $default: expr, $($rest:tt)* },)*) => { + $( + paste::paste!( + pub fn [<$field _opt>]() -> Option<$type> { $default } - )* + ); + )* + }; +} + +/// Define default value functions for those with `Some` default values. +macro_rules! def_default { + ($({ $field:ident, $type:ty, $default: expr, $($rest:tt)* },)*) => { + $( + def_default!(@ $field, $type, $default); + )* + }; + (@ $field:ident, $type:ty, None) => {}; + (@ $field:ident, $type:ty, $default: expr) => { + pub fn $field() -> $type { + $default.unwrap() } + paste::paste!( + pub static [<$field:upper>]: LazyLock<$type> = LazyLock::new($field); + ); }; } -for_all_params!(def_default); +/// Default values for all parameters. +pub mod default { + use std::sync::LazyLock; + + for_all_params!(def_default_opt); + for_all_params!(def_default); +} macro_rules! impl_check_missing_fields { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $($rest:tt)* },)*) => { /// Check if any undeprecated fields are missing. pub fn check_missing_params(params: &PbSystemParams) -> Result<()> { $( @@ -114,7 +167,7 @@ macro_rules! impl_check_missing_fields { /// Derive serialization to kv pairs. macro_rules! impl_system_params_to_kv { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $($rest:tt)* },)*) => { /// The returned map only contains undeprecated fields. /// Return error if there are missing fields. #[allow(clippy::vec_init_then_push)] @@ -131,7 +184,7 @@ macro_rules! impl_system_params_to_kv { } macro_rules! impl_derive_missing_fields { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $($rest:tt)* },)*) => { pub fn derive_missing_fields(params: &mut PbSystemParams) { $( if params.$field.is_none() && let Some(v) = OverrideFromParams::$field(params) { @@ -144,7 +197,7 @@ macro_rules! impl_derive_missing_fields { /// Derive deserialization from kv pairs. macro_rules! impl_system_params_from_kv { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $($rest:tt)* },)*) => { /// Try to deserialize deprecated fields as well. /// Return error if there are unrecognized fields. pub fn system_params_from_kv(mut kvs: Vec<(K, V)>) -> Result @@ -187,7 +240,7 @@ macro_rules! impl_system_params_from_kv { /// If you want custom rules, please override the default implementation in /// `OverrideValidateOnSet` below. macro_rules! impl_default_validation_on_set { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr, $($rest:tt)* },)*) => { #[allow(clippy::ptr_arg)] trait ValidateOnSet { $( @@ -236,7 +289,7 @@ macro_rules! impl_default_validation_on_set { /// /// Note that newer versions must be prioritized during derivation. macro_rules! impl_default_from_other_params { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $type:ty, $($rest:tt)* },)*) => { trait FromParams { $( fn $field(_params: &PbSystemParams) -> Option<$type> { @@ -248,35 +301,55 @@ macro_rules! impl_default_from_other_params { } macro_rules! impl_set_system_param { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { - /// Set a system parameter with the given value or default one, returns the new value. - pub fn set_system_param(params: &mut PbSystemParams, key: &str, value: Option) -> Result { - match key { + ($({ $field:ident, $type:ty, $default:expr, $($rest:tt)* },)*) => { + /// Set a system parameter with the given value or default one. + /// + /// Returns the new value if changed, or an error if the parameter is unrecognized + /// or the value is invalid. + pub fn set_system_param( + params: &mut PbSystemParams, + key: &str, + value: Option>, + ) -> Result> { + use crate::system_param::reader::{SystemParamsReader, SystemParamsRead}; + + match key { $( key_of!($field) => { let v = if let Some(v) = value { - v.parse().map_err(|_| format!("cannot parse parameter value"))? + v.as_ref().parse().map_err(|_| format!("cannot parse parameter value"))? } else { $default.ok_or_else(|| format!("{} does not have a default value", key))? }; OverrideValidateOnSet::$field(&v)?; - params.$field = Some(v.clone()); - return Ok(v.to_string()) + + let changed = SystemParamsReader::new(&*params).$field() != v; + if changed { + let new_value = v.to_string(); + let diff = SystemParamsDiff { + $field: Some(v.to_owned()), + ..Default::default() + }; + params.$field = Some(v); + Ok(Some((new_value, diff))) + } else { + Ok(None) + } }, )* _ => { - return Err(format!( - "unrecognized system param {:?}", + Err(format!( + "unrecognized system parameter {:?}", key - )); + )) } - }; + } } }; } macro_rules! impl_is_mutable { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr, $($rest:tt)* },)*) => { pub fn is_mutable(field: &str) -> Result { match field { $( @@ -289,7 +362,7 @@ macro_rules! impl_is_mutable { } macro_rules! impl_system_params_for_test { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $type:ty, $default:expr, $($rest:tt)* },)*) => { #[allow(clippy::needless_update)] pub fn system_params_for_test() -> PbSystemParams { let mut ret = PbSystemParams { @@ -300,6 +373,8 @@ macro_rules! impl_system_params_for_test { }; ret.data_directory = Some("hummock_001".to_string()); ret.state_store = Some("hummock+memory".to_string()); + ret.backup_storage_url = Some("memory".into()); + ret.backup_storage_directory = Some("backup".into()); ret } }; @@ -324,13 +399,17 @@ impl ValidateOnSet for OverrideValidateOnSet { Self::expect_range(*v, 1..) } - fn backup_storage_directory(_v: &String) -> Result<()> { - // TODO + fn backup_storage_directory(v: &String) -> Result<()> { + if v.trim().is_empty() { + return Err("backup_storage_directory cannot be empty".into()); + } Ok(()) } - fn backup_storage_url(_v: &String) -> Result<()> { - // TODO + fn backup_storage_url(v: &String) -> Result<()> { + if v.trim().is_empty() { + return Err("backup_storage_url cannot be empty".into()); + } Ok(()) } } @@ -360,7 +439,6 @@ mod tests { (BACKUP_STORAGE_DIRECTORY_KEY, "a"), (MAX_CONCURRENT_CREATING_STREAMING_JOBS_KEY, "1"), (PAUSE_ON_NEXT_BOOTSTRAP_KEY, "false"), - (WASM_STORAGE_URL_KEY, "a"), (ENABLE_TRACING_KEY, "true"), ("a_deprecated_param", "foo"), ]; @@ -382,7 +460,7 @@ mod tests { #[test] fn test_set() { - let mut p = PbSystemParams::default(); + let mut p = system_params_for_test(); // Unrecognized param. assert!(set_system_param(&mut p, "?", Some("?".to_string())).is_err()); // Value out of range. diff --git a/src/common/src/system_param/reader.rs b/src/common/src/system_param/reader.rs index 24ecd83f5f061..3374e72120238 100644 --- a/src/common/src/system_param/reader.rs +++ b/src/common/src/system_param/reader.rs @@ -12,87 +12,152 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::borrow::Borrow; + use risingwave_pb::meta::PbSystemParams; -use super::{default, system_params_to_kv}; +use super::{default, ParamValue}; +use crate::for_all_params; + +/// Information about a system parameter. +pub struct ParameterInfo { + pub name: &'static str, + pub mutable: bool, + pub value: String, + pub description: &'static str, +} -/// A wrapper for [`risingwave_pb::meta::SystemParams`] for 2 purposes: -/// - Avoid misuse of deprecated fields by hiding their getters. -/// - Abstract fallback logic for fields that might not be provided by meta service due to backward -/// compatibility. +macro_rules! define_system_params_read_trait { + ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr, $doc:literal, $($rest:tt)* },)*) => { + /// The trait delegating reads on [`risingwave_pb::meta::SystemParams`]. + /// + /// For 2 purposes: + /// - Avoid misuse of deprecated fields by hiding their getters. + /// - Abstract fallback logic for fields that might not be provided by meta service due to backward + /// compatibility. + pub trait SystemParamsRead { + $( + #[doc = $doc] + fn $field(&self) -> <$type as ParamValue>::Borrowed<'_>; + )* + + /// Return the information of all parameters. + fn get_all(&self) -> Vec { + vec![ + $( + ParameterInfo { + name: stringify!($field), + mutable: $is_mutable, + value: self.$field().to_string(), + description: $doc, + }, + )* + ] + } + } + }; +} + +for_all_params!(define_system_params_read_trait); + +/// The wrapper delegating reads on [`risingwave_pb::meta::SystemParams`]. +/// +/// See [`SystemParamsRead`] for more details. #[derive(Clone, Debug, PartialEq)] -pub struct SystemParamsReader { - prost: PbSystemParams, +pub struct SystemParamsReader { + inner: I, } -impl From for SystemParamsReader { - fn from(prost: PbSystemParams) -> Self { - Self { prost } +impl From for SystemParamsReader +where + I: Borrow, +{ + fn from(inner: I) -> Self { + Self { inner } } } -impl SystemParamsReader { - pub fn barrier_interval_ms(&self) -> u32 { - self.prost.barrier_interval_ms.unwrap() +impl SystemParamsReader +where + I: Borrow, +{ + pub fn new(inner: I) -> Self { + Self { inner } } - pub fn checkpoint_frequency(&self) -> u64 { - self.prost.checkpoint_frequency.unwrap() + /// Return a new reader with the reference to the inner system params. + pub fn as_ref(&self) -> SystemParamsReader<&PbSystemParams> { + SystemParamsReader { + inner: self.inner(), + } } - pub fn parallel_compact_size_mb(&self) -> u32 { - self.prost.parallel_compact_size_mb.unwrap() + fn inner(&self) -> &PbSystemParams { + self.inner.borrow() } +} - pub fn sstable_size_mb(&self) -> u32 { - self.prost.sstable_size_mb.unwrap() +/// - Unwrap the field if it always exists. +/// For example, if a parameter is introduced before the initial public release. +/// +/// - Otherwise, specify the fallback logic when the field is missing. +impl SystemParamsRead for SystemParamsReader +where + I: Borrow, +{ + fn barrier_interval_ms(&self) -> u32 { + self.inner().barrier_interval_ms.unwrap() } - pub fn block_size_kb(&self) -> u32 { - self.prost.block_size_kb.unwrap() + fn checkpoint_frequency(&self) -> u64 { + self.inner().checkpoint_frequency.unwrap() } - pub fn bloom_false_positive(&self) -> f64 { - self.prost.bloom_false_positive.unwrap() + fn parallel_compact_size_mb(&self) -> u32 { + self.inner().parallel_compact_size_mb.unwrap() } - pub fn state_store(&self) -> &str { - self.prost.state_store.as_ref().unwrap() + fn sstable_size_mb(&self) -> u32 { + self.inner().sstable_size_mb.unwrap() } - pub fn data_directory(&self) -> &str { - self.prost.data_directory.as_ref().unwrap() + fn block_size_kb(&self) -> u32 { + self.inner().block_size_kb.unwrap() } - pub fn backup_storage_url(&self) -> &str { - self.prost.backup_storage_url.as_ref().unwrap() + fn bloom_false_positive(&self) -> f64 { + self.inner().bloom_false_positive.unwrap() } - pub fn backup_storage_directory(&self) -> &str { - self.prost.backup_storage_directory.as_ref().unwrap() + fn state_store(&self) -> &str { + self.inner().state_store.as_ref().unwrap() } - pub fn max_concurrent_creating_streaming_jobs(&self) -> u32 { - self.prost.max_concurrent_creating_streaming_jobs.unwrap() + fn data_directory(&self) -> &str { + self.inner().data_directory.as_ref().unwrap() } - pub fn pause_on_next_bootstrap(&self) -> bool { - self.prost - .pause_on_next_bootstrap - .unwrap_or_else(|| default::pause_on_next_bootstrap().unwrap()) + fn backup_storage_url(&self) -> &str { + self.inner().backup_storage_url.as_ref().unwrap() } - pub fn enable_tracing(&self) -> bool { - self.prost - .enable_tracing - .unwrap_or_else(|| default::enable_tracing().unwrap()) + fn backup_storage_directory(&self) -> &str { + self.inner().backup_storage_directory.as_ref().unwrap() + } + + fn max_concurrent_creating_streaming_jobs(&self) -> u32 { + self.inner().max_concurrent_creating_streaming_jobs.unwrap() } - pub fn wasm_storage_url(&self) -> &str { - self.prost.wasm_storage_url.as_ref().unwrap() + fn pause_on_next_bootstrap(&self) -> bool { + self.inner() + .pause_on_next_bootstrap + .unwrap_or_else(default::pause_on_next_bootstrap) } - pub fn to_kv(&self) -> Vec<(String, String)> { - system_params_to_kv(&self.prost).unwrap() + fn enable_tracing(&self) -> bool { + self.inner() + .enable_tracing + .unwrap_or_else(default::enable_tracing) } } diff --git a/src/common/src/telemetry/mod.rs b/src/common/src/telemetry/mod.rs index e25e312013593..382e50d8eb927 100644 --- a/src/common/src/telemetry/mod.rs +++ b/src/common/src/telemetry/mod.rs @@ -17,6 +17,11 @@ pub mod report; use std::time::SystemTime; +use risingwave_pb::telemetry::{ + ReportBase as PbTelemetryReportBase, SystemCpu as PbSystemCpu, SystemData as PbSystemData, + SystemMemory as PbSystemMemory, SystemOs as PbSystemOs, + TelemetryNodeType as PbTelemetryNodeType, +}; use serde::{Deserialize, Serialize}; use sysinfo::System; use thiserror_ext::AsReport; @@ -65,6 +70,19 @@ pub struct TelemetryReportBase { pub node_type: TelemetryNodeType, } +impl From for PbTelemetryReportBase { + fn from(val: TelemetryReportBase) -> Self { + PbTelemetryReportBase { + tracking_id: val.tracking_id, + session_id: val.session_id, + system_data: Some(val.system_data.into()), + up_time: val.up_time, + report_time: val.time_stamp, + node_type: from_telemetry_node_type(val.node_type) as i32, + } + } +} + pub trait TelemetryReport: Serialize {} #[derive(Debug, Serialize, Deserialize)] @@ -155,6 +173,63 @@ pub fn current_timestamp() -> u64 { .as_secs() } +fn from_telemetry_node_type(t: TelemetryNodeType) -> PbTelemetryNodeType { + match t { + TelemetryNodeType::Meta => PbTelemetryNodeType::Meta, + TelemetryNodeType::Compute => PbTelemetryNodeType::Compute, + TelemetryNodeType::Frontend => PbTelemetryNodeType::Frontend, + TelemetryNodeType::Compactor => PbTelemetryNodeType::Compactor, + } +} + +impl From for PbTelemetryNodeType { + fn from(val: TelemetryNodeType) -> Self { + match val { + TelemetryNodeType::Meta => PbTelemetryNodeType::Meta, + TelemetryNodeType::Compute => PbTelemetryNodeType::Compute, + TelemetryNodeType::Frontend => PbTelemetryNodeType::Frontend, + TelemetryNodeType::Compactor => PbTelemetryNodeType::Compactor, + } + } +} + +impl From for PbSystemCpu { + fn from(val: Cpu) -> Self { + PbSystemCpu { + available: val.available, + } + } +} + +impl From for PbSystemMemory { + fn from(val: Memory) -> Self { + PbSystemMemory { + used: val.used as u64, + total: val.total as u64, + } + } +} + +impl From for PbSystemOs { + fn from(val: Os) -> Self { + PbSystemOs { + name: val.name, + kernel_version: val.kernel_version, + version: val.version, + } + } +} + +impl From for PbSystemData { + fn from(val: SystemData) -> Self { + PbSystemData { + memory: Some(val.memory.into()), + os: Some(val.os.into()), + cpu: Some(val.cpu.into()), + } + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/common/src/types/datetime.rs b/src/common/src/types/datetime.rs index bc18858d678ec..c609017d06e3f 100644 --- a/src/common/src/types/datetime.rs +++ b/src/common/src/types/datetime.rs @@ -328,17 +328,15 @@ impl ToText for Date { /// ``` fn write(&self, f: &mut W) -> std::fmt::Result { let (ce, year) = self.0.year_ce(); - if ce { - write!(f, "{}", self.0) - } else { - write!( - f, - "{:04}-{:02}-{:02} BC", - year, - self.0.month(), - self.0.day() - ) - } + let suffix = if ce { "" } else { " BC" }; + write!( + f, + "{:04}-{:02}-{:02}{}", + year, + self.0.month(), + self.0.day(), + suffix + ) } fn write_with_type(&self, ty: &DataType, f: &mut W) -> std::fmt::Result { @@ -364,7 +362,17 @@ impl ToText for Time { impl ToText for Timestamp { fn write(&self, f: &mut W) -> std::fmt::Result { - write!(f, "{}", self.0) + let (ce, year) = self.0.year_ce(); + let suffix = if ce { "" } else { " BC" }; + write!( + f, + "{:04}-{:02}-{:02} {}{}", + year, + self.0.month(), + self.0.day(), + self.0.time(), + suffix + ) } fn write_with_type(&self, ty: &DataType, f: &mut W) -> std::fmt::Result { @@ -536,6 +544,12 @@ impl Timestamp { self.0.timestamp_nanos_opt().unwrap() } + pub fn with_millis(timestamp_millis: i64) -> Result { + let secs = timestamp_millis.div_euclid(1_000); + let nsecs = timestamp_millis.rem_euclid(1_000) * 1_000_000; + Self::with_secs_nsecs(secs, nsecs as u32) + } + pub fn with_micros(timestamp_micros: i64) -> Result { let secs = timestamp_micros.div_euclid(1_000_000); let nsecs = timestamp_micros.rem_euclid(1_000_000) * 1000; diff --git a/src/common/src/types/fields.rs b/src/common/src/types/fields.rs index 0e5912ead0dc4..df1795804af00 100644 --- a/src/common/src/types/fields.rs +++ b/src/common/src/types/fields.rs @@ -12,13 +12,71 @@ // See the License for the specific language governing permissions and // limitations under the License. use super::DataType; +use crate::row::OwnedRow; +use crate::util::chunk_coalesce::DataChunkBuilder; /// A struct can implements `Fields` when if can be represented as a relational Row. /// -/// Can be automatically derived with [`#[derive(Fields)]`](derive@super::Fields). +/// # Derivable +/// +/// This trait can be automatically derived with [`#[derive(Fields)]`](derive@super::Fields). +/// Type of the fields must implement [`WithDataType`](super::WithDataType) and [`ToOwnedDatum`](super::ToOwnedDatum). +/// +/// ``` +/// # use risingwave_common::types::Fields; +/// +/// #[derive(Fields)] +/// struct Data { +/// v1: i16, +/// v2: i32, +/// } +/// ``` +/// +/// You can add `#[primary_key]` attribute to one of the fields to specify the primary key of the table. +/// +/// ``` +/// # use risingwave_common::types::Fields; +/// +/// #[derive(Fields)] +/// struct Data { +/// #[primary_key] +/// v1: i16, +/// v2: i32, +/// } +/// ``` +/// +/// If the primary key is composite, you can add `#[primary_key(...)]` attribute to the struct to specify the order of the fields. +/// +/// ``` +/// # use risingwave_common::types::Fields; +/// +/// #[derive(Fields)] +/// #[primary_key(v2, v1)] +/// struct Data { +/// v1: i16, +/// v2: i32, +/// } +/// ``` pub trait Fields { - /// When the struct being converted to an [`Row`](crate::row::Row) or a [`DataChunk`](crate::array::DataChunk), it schema must be consistent with the `fields` call. + /// The primary key of the table. + /// + /// - `None` if the primary key is not applicable. + /// - `Some(&[])` if the primary key is empty, i.e., there'll be at most one row in the table. + const PRIMARY_KEY: Option<&'static [usize]>; + + /// Return the schema of the struct. fn fields() -> Vec<(&'static str, DataType)>; + + /// Convert the struct to an `OwnedRow`. + fn into_owned_row(self) -> OwnedRow; + + /// Create a [`DataChunkBuilder`](crate::util::chunk_coalesce::DataChunkBuilder) with the schema of the struct. + fn data_chunk_builder(capacity: usize) -> DataChunkBuilder { + DataChunkBuilder::new( + Self::fields().into_iter().map(|(_, ty)| ty).collect(), + capacity, + ) + } } #[cfg(test)] @@ -48,8 +106,8 @@ mod tests { v7: Vec, v8: std::vec::Vec, v9: Option>, - v10: std::option::Option>>, - v11: Box, + v10: std::option::Option>>, + v11: Timestamp, v14: Sub, } diff --git a/src/common/src/types/interval.rs b/src/common/src/types/interval.rs index 9224c8955db66..0160ff7955591 100644 --- a/src/common/src/types/interval.rs +++ b/src/common/src/types/interval.rs @@ -384,6 +384,11 @@ impl Interval { self > &Self::from_month_day_usec(0, 0, 0) } + /// Checks if all fields of [`Interval`] are all non-negative. + pub fn is_never_negative(&self) -> bool { + self.months >= 0 && self.days >= 0 && self.usecs >= 0 + } + /// Truncate the interval to the precision of milliseconds. /// /// # Example @@ -1051,7 +1056,7 @@ impl Interval { pub fn from_iso_8601(s: &str) -> ParseResult { // ISO pattern - PnYnMnDTnHnMnS static ISO_8601_REGEX: LazyLock = LazyLock::new(|| { - Regex::new(r"P([0-9]+)Y([0-9]+)M([0-9]+)DT([0-9]+)H([0-9]+)M([0-9]+(?:\.[0-9]+)?)S") + Regex::new(r"^P([0-9]+)Y([0-9]+)M([0-9]+)DT([0-9]+)H([0-9]+)M([0-9]+(?:\.[0-9]+)?)S$") .unwrap() }); // wrap into a closure to simplify error handling @@ -1453,7 +1458,10 @@ impl Interval { if let Some(leading_field) = leading_field { Self::parse_sql_standard(s, leading_field) } else { - Self::parse_postgres(s) + match s.as_bytes().get(0) { + Some(b'P') => Self::from_iso_8601(s), + _ => Self::parse_postgres(s), + } } } } @@ -1530,6 +1538,12 @@ mod tests { interval, Interval::from_month(14) + Interval::from_days(3) + Interval::from_minutes(62) ); + + let interval = "P1Y2M3DT0H5M0S".parse::().unwrap(); + assert_eq!( + interval, + Interval::from_month(14) + Interval::from_days(3) + Interval::from_minutes(5) + ); } #[test] diff --git a/src/common/src/types/mod.rs b/src/common/src/types/mod.rs index 0b80f57e132b2..71f352ce2f87d 100644 --- a/src/common/src/types/mod.rs +++ b/src/common/src/types/mod.rs @@ -293,6 +293,36 @@ impl From for PbTypeName { } } +/// Convenient macros to generate match arms for [`DataType`](crate::types::DataType). +pub mod data_types { + /// Numeric [`DataType`](crate::types::DataType)s supported to be `offset` of `RANGE` frame. + #[macro_export] + macro_rules! range_frame_numeric { + () => { + DataType::Int16 + | DataType::Int32 + | DataType::Int64 + | DataType::Float32 + | DataType::Float64 + | DataType::Decimal + }; + } + pub use range_frame_numeric; + + /// Date/time [`DataType`](crate::types::DataType)s supported to be `offset` of `RANGE` frame. + #[macro_export] + macro_rules! range_frame_datetime { + () => { + DataType::Date + | DataType::Time + | DataType::Timestamp + | DataType::Timestamptz + | DataType::Interval + }; + } + pub use range_frame_datetime; +} + impl DataType { pub fn create_array_builder(&self, capacity: usize) -> ArrayBuilderImpl { use crate::array::*; @@ -374,7 +404,7 @@ impl DataType { matches!(self, DataType::Int16 | DataType::Int32 | DataType::Int64) } - /// Returns the output type of window function on a given input type. + /// Returns the output type of time window function on a given input type. pub fn window_of(input: &DataType) -> Option { match input { DataType::Timestamptz => Some(DataType::Timestamptz), @@ -570,10 +600,24 @@ pub trait ToOwnedDatum { fn to_owned_datum(self) -> Datum; } -impl ToOwnedDatum for DatumRef<'_> { +impl ToOwnedDatum for &Datum { #[inline(always)] fn to_owned_datum(self) -> Datum { - self.map(ScalarRefImpl::into_scalar_impl) + self.clone() + } +} + +impl> ToOwnedDatum for T { + #[inline(always)] + fn to_owned_datum(self) -> Datum { + Some(self.into()) + } +} + +impl> ToOwnedDatum for Option { + #[inline(always)] + fn to_owned_datum(self) -> Datum { + self.map(Into::into) } } @@ -765,6 +809,36 @@ impl From> for ScalarImpl { } } +impl From> for ScalarImpl { + fn from(v: Vec) -> Self { + Self::List(v.into_iter().collect()) + } +} + +impl From>> for ScalarImpl { + fn from(v: Vec>) -> Self { + Self::List(v.into_iter().collect()) + } +} + +impl From> for ScalarImpl { + fn from(v: Vec) -> Self { + Self::List(v.iter().map(|s| s.as_str()).collect()) + } +} + +impl From> for ScalarImpl { + fn from(v: Vec) -> Self { + Self::Bytea(v.into()) + } +} + +impl From for ScalarImpl { + fn from(v: Bytes) -> Self { + Self::Bytea(v.as_ref().into()) + } +} + impl ScalarImpl { /// Creates a scalar from binary. pub fn from_binary(bytes: &Bytes, data_type: &DataType) -> Result { diff --git a/src/common/src/types/sentinel.rs b/src/common/src/types/sentinel.rs index 864ec4ca54d67..22eb91d4d476b 100644 --- a/src/common/src/types/sentinel.rs +++ b/src/common/src/types/sentinel.rs @@ -37,6 +37,38 @@ impl Sentinelled { pub fn is_sentinel(&self) -> bool { matches!(self, Self::Smallest | Self::Largest) } + + pub fn cmp_by( + &self, + other: &Self, + cmp_fn: impl FnOnce(&T, &T) -> std::cmp::Ordering, + ) -> std::cmp::Ordering { + use Sentinelled::*; + match (self, other) { + (Smallest, Smallest) => std::cmp::Ordering::Equal, + (Smallest, _) => std::cmp::Ordering::Less, + (_, Smallest) => std::cmp::Ordering::Greater, + (Largest, Largest) => std::cmp::Ordering::Equal, + (Largest, _) => std::cmp::Ordering::Greater, + (_, Largest) => std::cmp::Ordering::Less, + (Normal(a), Normal(b)) => cmp_fn(a, b), + } + } + + pub fn map(self, map_fn: impl FnOnce(T) -> U) -> Sentinelled { + use Sentinelled::*; + match self { + Smallest => Smallest, + Normal(inner) => Normal(map_fn(inner)), + Largest => Largest, + } + } +} + +impl From for Sentinelled { + fn from(inner: T) -> Self { + Self::Normal(inner) + } } impl PartialOrd for Sentinelled @@ -53,16 +85,7 @@ where T: Ord, { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - use Sentinelled::*; - match (self, other) { - (Smallest, Smallest) => std::cmp::Ordering::Equal, - (Smallest, _) => std::cmp::Ordering::Less, - (_, Smallest) => std::cmp::Ordering::Greater, - (Largest, Largest) => std::cmp::Ordering::Equal, - (Largest, _) => std::cmp::Ordering::Greater, - (_, Largest) => std::cmp::Ordering::Less, - (Normal(a), Normal(b)) => a.cmp(b), - } + self.cmp_by(other, T::cmp) } } diff --git a/src/common/src/types/serial.rs b/src/common/src/types/serial.rs index 9bfbf5e4fcac7..5c84c95fa0f7a 100644 --- a/src/common/src/types/serial.rs +++ b/src/common/src/types/serial.rs @@ -26,6 +26,12 @@ use crate::util::row_id::RowId; #[derive(Debug, Copy, Clone, PartialEq, Eq, Ord, PartialOrd, Default, Hash)] pub struct Serial(i64); +impl From for i64 { + fn from(value: Serial) -> i64 { + value.0 + } +} + impl From for Serial { fn from(value: i64) -> Self { Self(value) diff --git a/src/common/src/types/timestamptz.rs b/src/common/src/types/timestamptz.rs index 41359bdf84377..f14d5d2edee6e 100644 --- a/src/common/src/types/timestamptz.rs +++ b/src/common/src/types/timestamptz.rs @@ -17,7 +17,7 @@ use std::io::Write; use std::str::FromStr; use bytes::{Bytes, BytesMut}; -use chrono::{TimeZone, Utc}; +use chrono::{DateTime, Datelike, TimeZone, Utc}; use chrono_tz::Tz; use postgres_types::{accepts, to_sql_checked, IsNull, ToSql, Type}; use serde::{Deserialize, Serialize}; @@ -201,6 +201,26 @@ impl std::fmt::Display for Timestamptz { } } +pub fn write_date_time_tz( + instant_local: DateTime, + writer: &mut impl std::fmt::Write, +) -> std::fmt::Result { + let date = instant_local.date_naive(); + let (ce, year) = date.year_ce(); + write!( + writer, + "{:04}-{:02}-{:02} {}", + year, + date.month(), + date.day(), + instant_local.format(if ce { + "%H:%M:%S%.f%:z" + } else { + "%H:%M:%S%.f%:z BC" + }) + ) +} + #[cfg(test)] mod test { use super::*; diff --git a/src/common/src/util/chunk_coalesce.rs b/src/common/src/util/chunk_coalesce.rs index d57ebc2800479..5cdeb8026a0f1 100644 --- a/src/common/src/util/chunk_coalesce.rs +++ b/src/common/src/util/chunk_coalesce.rs @@ -143,6 +143,11 @@ impl DataChunkBuilder { } } + /// Build a data chunk from the current buffer. + pub fn finish(mut self) -> DataChunk { + self.build_data_chunk() + } + fn append_one_row_internal(&mut self, data_chunk: &DataChunk, row_idx: usize) { self.do_append_one_row_from_datums(data_chunk.row_at(row_idx).0.iter()); } @@ -456,13 +461,13 @@ mod tests { for v in [1, 2, 3, 4, 5] { left_array_builder.append(&Some(ScalarImpl::Int32(v))); } - let left_arrays = vec![left_array_builder.finish()]; + let left_arrays = [left_array_builder.finish()]; let mut right_array_builder = DataType::Int64.create_array_builder(5); for v in [5, 4, 3, 2, 1] { right_array_builder.append(&Some(ScalarImpl::Int64(v))); } - let right_arrays = vec![right_array_builder.finish()]; + let right_arrays = [right_array_builder.finish()]; let mut output_chunks = Vec::new(); diff --git a/src/common/src/util/mod.rs b/src/common/src/util/mod.rs index a629512b3d63e..917d982be3db6 100644 --- a/src/common/src/util/mod.rs +++ b/src/common/src/util/mod.rs @@ -21,7 +21,6 @@ pub mod compress; pub mod deployment; pub mod env_var; pub mod epoch; -mod future_utils; pub mod hash_util; pub mod iter_util; pub mod memcmp_encoding; @@ -42,9 +41,5 @@ pub mod tracing; pub mod value_encoding; pub mod worker_util; -pub use future_utils::{ - await_future_with_monitor_error_stream, drop_either_future, pending_on_none, select_all, - RwFutureExt, RwTryStreamExt, -}; #[macro_use] pub mod match_util; diff --git a/src/common/src/util/scan_range.rs b/src/common/src/util/scan_range.rs index df42fa9239070..19e0cb50b83c6 100644 --- a/src/common/src/util/scan_range.rs +++ b/src/common/src/util/scan_range.rs @@ -19,7 +19,7 @@ use risingwave_pb::batch_plan::scan_range::Bound as BoundPb; use risingwave_pb::batch_plan::ScanRange as ScanRangePb; use super::value_encoding::serialize_datum; -use crate::catalog::get_dist_key_in_pk_indices; +use crate::hash::table_distribution::TableDistribution; use crate::hash::VirtualNode; use crate::types::{Datum, ScalarImpl}; use crate::util::value_encoding::serialize_datum_into; @@ -83,31 +83,8 @@ impl ScanRange { } } - pub fn try_compute_vnode( - &self, - dist_key_indices: &[usize], - pk_indices: &[usize], - ) -> Option { - if dist_key_indices.is_empty() { - return Some(VirtualNode::ZERO); - } - - let dist_key_in_pk_indices = get_dist_key_in_pk_indices(dist_key_indices, pk_indices); - self.try_compute_vnode_with_dist_key_in_pk_indices(&dist_key_in_pk_indices) - } - - pub fn try_compute_vnode_with_dist_key_in_pk_indices( - &self, - dist_key_in_pk_indices: &[usize], - ) -> Option { - let pk_prefix_len = self.eq_conds.len(); - if dist_key_in_pk_indices.iter().any(|&i| i >= pk_prefix_len) { - return None; - } - - let pk_prefix_value: &[_] = &self.eq_conds; - let vnode = VirtualNode::compute_row(pk_prefix_value, dist_key_in_pk_indices); - Some(vnode) + pub fn try_compute_vnode(&self, table_distribution: &TableDistribution) -> Option { + table_distribution.try_compute_vnode_by_pk_prefix(self.eq_conds.as_slice()) } } @@ -180,12 +157,15 @@ mod tests { fn test_vnode_prefix() { let dist_key = vec![1, 3]; let pk = vec![1, 3, 2]; + let dist_key_idx_in_pk = + crate::catalog::get_dist_key_in_pk_indices(&dist_key, &pk).unwrap(); + let dist = TableDistribution::all(dist_key_idx_in_pk); let mut scan_range = ScanRange::full_table_scan(); - assert!(scan_range.try_compute_vnode(&dist_key, &pk).is_none()); + assert!(scan_range.try_compute_vnode(&dist).is_none()); scan_range.eq_conds.push(Some(ScalarImpl::from(114))); - assert!(scan_range.try_compute_vnode(&dist_key, &pk).is_none()); + assert!(scan_range.try_compute_vnode(&dist).is_none()); scan_range.eq_conds.push(Some(ScalarImpl::from(514))); let row = OwnedRow::new(vec![ @@ -195,7 +175,7 @@ mod tests { let vnode = VirtualNode::compute_row(&row, &[0, 1]); - assert_eq!(scan_range.try_compute_vnode(&dist_key, &pk), Some(vnode)); + assert_eq!(scan_range.try_compute_vnode(&dist), Some(vnode)); } // dist_key is not prefix of pk @@ -203,15 +183,18 @@ mod tests { fn test_vnode_not_prefix() { let dist_key = vec![2, 3]; let pk = vec![1, 3, 2]; + let dist_key_idx_in_pk = + crate::catalog::get_dist_key_in_pk_indices(&dist_key, &pk).unwrap(); + let dist = TableDistribution::all(dist_key_idx_in_pk); let mut scan_range = ScanRange::full_table_scan(); - assert!(scan_range.try_compute_vnode(&dist_key, &pk).is_none()); + assert!(scan_range.try_compute_vnode(&dist).is_none()); scan_range.eq_conds.push(Some(ScalarImpl::from(114))); - assert!(scan_range.try_compute_vnode(&dist_key, &pk).is_none()); + assert!(scan_range.try_compute_vnode(&dist).is_none()); scan_range.eq_conds.push(Some(ScalarImpl::from(514))); - assert!(scan_range.try_compute_vnode(&dist_key, &pk).is_none()); + assert!(scan_range.try_compute_vnode(&dist).is_none()); scan_range.eq_conds.push(Some(ScalarImpl::from(114514))); let row = OwnedRow::new(vec![ @@ -222,6 +205,6 @@ mod tests { let vnode = VirtualNode::compute_row(&row, &[2, 1]); - assert_eq!(scan_range.try_compute_vnode(&dist_key, &pk), Some(vnode)); + assert_eq!(scan_range.try_compute_vnode(&dist), Some(vnode)); } } diff --git a/src/common/src/util/sort_util.rs b/src/common/src/util/sort_util.rs index 9c2a8c05b6282..42d548354581f 100644 --- a/src/common/src/util/sort_util.rs +++ b/src/common/src/util/sort_util.rs @@ -510,9 +510,7 @@ pub fn partial_cmp_datum_iter( ) -> Option { let mut order_types_iter = order_types.into_iter(); lhs.into_iter().partial_cmp_by(rhs, |x, y| { - let Some(order_type) = order_types_iter.next() else { - return None; - }; + let order_type = order_types_iter.next()?; partial_cmp_datum(x, y, order_type) }) } diff --git a/src/common/src/util/tracing.rs b/src/common/src/util/tracing.rs index e7da6e8e7d580..19523dde56324 100644 --- a/src/common/src/util/tracing.rs +++ b/src/common/src/util/tracing.rs @@ -19,7 +19,7 @@ use std::pin::Pin; use std::task::{Context, Poll}; use opentelemetry::propagation::TextMapPropagator; -use opentelemetry::sdk::propagation::TraceContextPropagator; +use opentelemetry_sdk::propagation::TraceContextPropagator; use tracing_opentelemetry::OpenTelemetrySpanExt; /// Context for tracing used for propagating tracing information in a distributed system. diff --git a/src/compute/Cargo.toml b/src/compute/Cargo.toml index fecb5738c246a..e3f8812d57025 100644 --- a/src/compute/Cargo.toml +++ b/src/compute/Cargo.toml @@ -32,11 +32,11 @@ risingwave_common = { workspace = true } risingwave_common_heap_profiling = { workspace = true } risingwave_common_service = { workspace = true } risingwave_connector = { workspace = true } +risingwave_dml = { workspace = true } risingwave_hummock_sdk = { workspace = true } risingwave_jni_core = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } -risingwave_source = { workspace = true } risingwave_storage = { workspace = true } risingwave_stream = { workspace = true } serde = { version = "1", features = ["derive"] } diff --git a/src/compute/src/lib.rs b/src/compute/src/lib.rs index 342448066dd06..3673997d2a128 100644 --- a/src/compute/src/lib.rs +++ b/src/compute/src/lib.rs @@ -64,6 +64,8 @@ pub struct ComputeNodeOpts { #[clap(long, env = "RW_ADVERTISE_ADDR")] pub advertise_addr: Option, + /// We will start a http server at this address via `MetricsManager`. + /// Then the prometheus instance will poll the metrics from this address. #[clap( long, env = "RW_PROMETHEUS_LISTENER_ADDR", @@ -92,10 +94,6 @@ pub struct ComputeNodeOpts { #[clap(long, env = "RW_TOTAL_MEMORY_BYTES", default_value_t = default_total_memory_bytes())] pub total_memory_bytes: usize, - /// Spill threshold for mem table. - #[clap(long, env = "RW_MEM_TABLE_SPILL_THRESHOLD", default_value_t = default_mem_table_spill_threshold())] - pub mem_table_spill_threshold: usize, - /// The parallelism that the compute node will register to the scheduler of the meta service. #[clap(long, env = "RW_PARALLELISM", default_value_t = default_parallelism())] #[override_opts(if_absent, path = streaming.actor_runtime_worker_threads_num)] @@ -227,18 +225,14 @@ pub fn start(opts: ComputeNodeOpts) -> Pin + Send>> }) } -fn default_total_memory_bytes() -> usize { +pub fn default_total_memory_bytes() -> usize { (system_memory_available_bytes() as f64 * DEFAULT_MEMORY_PROPORTION) as usize } -fn default_mem_table_spill_threshold() -> usize { - (4 << 20) as usize -} - -fn default_parallelism() -> usize { +pub fn default_parallelism() -> usize { total_cpu_available().ceil() as usize } -fn default_role() -> Role { +pub fn default_role() -> Role { Role::Both } diff --git a/src/compute/src/memory/controller.rs b/src/compute/src/memory/controller.rs index 683593b0f2a4c..a1695bdc565be 100644 --- a/src/compute/src/memory/controller.rs +++ b/src/compute/src/memory/controller.rs @@ -99,21 +99,30 @@ impl std::fmt::Debug for LruWatermarkController { /// /// - `stats.allocated`: Total number of bytes allocated by the application. /// - `stats.active`: Total number of bytes in active pages allocated by the application. This is a multiple of the page size, and greater than or equal to `stats.allocated`. This does not include `stats.arenas..pdirty`, `stats.arenas..pmuzzy`, nor pages entirely devoted to allocator metadata. +/// - `stats.resident`: Total number of bytes in physically resident data pages mapped by the allocator. +/// - `stats.metadata`: Total number of bytes dedicated to jemalloc metadata. /// /// Reference: -fn jemalloc_memory_stats() -> (usize, usize) { +fn jemalloc_memory_stats() -> (usize, usize, usize, usize) { if let Err(e) = tikv_jemalloc_ctl::epoch::advance() { tracing::warn!("Jemalloc epoch advance failed! {:?}", e); } let allocated = tikv_jemalloc_ctl::stats::allocated::read().unwrap(); let active = tikv_jemalloc_ctl::stats::active::read().unwrap(); - (allocated, active) + let resident = tikv_jemalloc_ctl::stats::resident::read().unwrap(); + let metadata = tikv_jemalloc_ctl::stats::metadata::read().unwrap(); + (allocated, active, resident, metadata) } impl LruWatermarkController { pub fn tick(&mut self, interval_ms: u32) -> Epoch { // NOTE: Be careful! The meaning of `allocated` and `active` differ in JeMalloc and JVM - let (jemalloc_allocated_bytes, jemalloc_active_bytes) = jemalloc_memory_stats(); + let ( + jemalloc_allocated_bytes, + jemalloc_active_bytes, + jemalloc_resident_bytes, + jemalloc_metadata_bytes, + ) = jemalloc_memory_stats(); let (jvm_allocated_bytes, jvm_active_bytes) = load_jvm_memory_stats(); let cur_used_memory_bytes = jemalloc_active_bytes + jvm_allocated_bytes; @@ -188,6 +197,12 @@ impl LruWatermarkController { self.metrics .jemalloc_active_bytes .set(jemalloc_active_bytes as i64); + self.metrics + .jemalloc_resident_bytes + .set(jemalloc_resident_bytes as i64); + self.metrics + .jemalloc_metadata_bytes + .set(jemalloc_metadata_bytes as i64); self.metrics .jvm_allocated_bytes .set(jvm_allocated_bytes as i64); diff --git a/src/compute/src/memory/manager.rs b/src/compute/src/memory/manager.rs index 1df90ba1b1cac..247ff4601be01 100644 --- a/src/compute/src/memory/manager.rs +++ b/src/compute/src/memory/manager.rs @@ -17,6 +17,7 @@ use std::sync::{Arc, Mutex}; use std::time::Duration; use risingwave_common::system_param::local_manager::SystemParamsReaderRef; +use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_stream::executor::monitor::StreamingMetrics; use super::controller::LruWatermarkController; diff --git a/src/compute/src/rpc/service/config_service.rs b/src/compute/src/rpc/service/config_service.rs index 917df842be4be..b5964c26349e3 100644 --- a/src/compute/src/rpc/service/config_service.rs +++ b/src/compute/src/rpc/service/config_service.rs @@ -23,7 +23,7 @@ use tonic::{Code, Request, Response, Status}; pub struct ConfigServiceImpl { batch_mgr: Arc, - stream_mgr: Arc, + stream_mgr: LocalStreamManager, } #[async_trait::async_trait] @@ -34,7 +34,7 @@ impl ConfigService for ConfigServiceImpl { ) -> Result, Status> { let batch_config = serde_json::to_string(self.batch_mgr.config()) .map_err(|e| e.to_status(Code::Internal, "compute"))?; - let stream_config = serde_json::to_string(&self.stream_mgr.config().await) + let stream_config = serde_json::to_string(&self.stream_mgr.env.config()) .map_err(|e| e.to_status(Code::Internal, "compute"))?; let show_config_response = ShowConfigResponse { @@ -46,7 +46,7 @@ impl ConfigService for ConfigServiceImpl { } impl ConfigServiceImpl { - pub fn new(batch_mgr: Arc, stream_mgr: Arc) -> Self { + pub fn new(batch_mgr: Arc, stream_mgr: LocalStreamManager) -> Self { Self { batch_mgr, stream_mgr, diff --git a/src/compute/src/rpc/service/exchange_service.rs b/src/compute/src/rpc/service/exchange_service.rs index d8b74be65f69d..2652daa9b8250 100644 --- a/src/compute/src/rpc/service/exchange_service.rs +++ b/src/compute/src/rpc/service/exchange_service.rs @@ -38,7 +38,7 @@ const BATCH_EXCHANGE_BUFFER_SIZE: usize = 1024; #[derive(Clone)] pub struct ExchangeServiceImpl { batch_mgr: Arc, - stream_mgr: Arc, + stream_mgr: LocalStreamManager, metrics: Arc, } @@ -128,7 +128,7 @@ impl ExchangeService for ExchangeServiceImpl { impl ExchangeServiceImpl { pub fn new( mgr: Arc, - stream_mgr: Arc, + stream_mgr: LocalStreamManager, metrics: Arc, ) -> Self { ExchangeServiceImpl { diff --git a/src/compute/src/rpc/service/monitor_service.rs b/src/compute/src/rpc/service/monitor_service.rs index 51dfa6dde0440..0d822fd3597f6 100644 --- a/src/compute/src/rpc/service/monitor_service.rs +++ b/src/compute/src/rpc/service/monitor_service.rs @@ -15,33 +15,35 @@ use std::ffi::CString; use std::fs; use std::path::Path; -use std::sync::Arc; use std::time::Duration; use itertools::Itertools; -use risingwave_common::config::ServerConfig; +use prometheus::core::Collector; +use risingwave_common::config::{MetricLevel, ServerConfig}; use risingwave_common_heap_profiling::{AUTO_DUMP_SUFFIX, COLLAPSED_SUFFIX, MANUALLY_DUMP_SUFFIX}; use risingwave_pb::monitor_service::monitor_service_server::MonitorService; use risingwave_pb::monitor_service::{ - AnalyzeHeapRequest, AnalyzeHeapResponse, HeapProfilingRequest, HeapProfilingResponse, - ListHeapProfilingRequest, ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, - StackTraceRequest, StackTraceResponse, + AnalyzeHeapRequest, AnalyzeHeapResponse, BackPressureInfo, GetBackPressureRequest, + GetBackPressureResponse, HeapProfilingRequest, HeapProfilingResponse, ListHeapProfilingRequest, + ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, StackTraceRequest, + StackTraceResponse, }; use risingwave_rpc_client::error::ToTonicStatus; +use risingwave_stream::executor::monitor::global_streaming_metrics; use risingwave_stream::task::LocalStreamManager; use thiserror_ext::AsReport; use tonic::{Code, Request, Response, Status}; #[derive(Clone)] pub struct MonitorServiceImpl { - stream_mgr: Arc, + stream_mgr: LocalStreamManager, grpc_await_tree_reg: Option, server_config: ServerConfig, } impl MonitorServiceImpl { pub fn new( - stream_mgr: Arc, + stream_mgr: LocalStreamManager, grpc_await_tree_reg: Option, server_config: ServerConfig, ) -> Self { @@ -65,7 +67,6 @@ impl MonitorService for MonitorServiceImpl { let actor_traces = self .stream_mgr .get_actor_traces() - .await .into_iter() .map(|(k, v)| (k, v.to_string())) .collect(); @@ -230,6 +231,39 @@ impl MonitorService for MonitorServiceImpl { let file = fs::read(Path::new(&collapsed_path_str))?; Ok(Response::new(AnalyzeHeapResponse { result: file })) } + + #[cfg_attr(coverage, coverage(off))] + async fn get_back_pressure( + &self, + _request: Request, + ) -> Result, Status> { + let metric_family = global_streaming_metrics(MetricLevel::Info) + .actor_output_buffer_blocking_duration_ns + .collect(); + let metrics = metric_family.get(0).unwrap().get_metric(); + let mut back_pressure_infos: Vec = Vec::new(); + for label_pairs in metrics { + let mut back_pressure_info = BackPressureInfo::default(); + for label_pair in label_pairs.get_label() { + if label_pair.get_name() == "actor_id" { + back_pressure_info.actor_id = label_pair.get_value().parse::().unwrap(); + } + if label_pair.get_name() == "fragment_id" { + back_pressure_info.fragment_id = label_pair.get_value().parse::().unwrap(); + } + if label_pair.get_name() == "downstream_fragment_id" { + back_pressure_info.downstream_fragment_id = + label_pair.get_value().parse::().unwrap(); + } + } + back_pressure_info.value = label_pairs.get_counter().get_value(); + back_pressure_infos.push(back_pressure_info); + } + + Ok(Response::new(GetBackPressureResponse { + back_pressure_infos, + })) + } } pub use grpc_middleware::*; diff --git a/src/compute/src/rpc/service/stream_service.rs b/src/compute/src/rpc/service/stream_service.rs index 22da99e58a414..6e96406743f29 100644 --- a/src/compute/src/rpc/service/stream_service.rs +++ b/src/compute/src/rpc/service/stream_service.rs @@ -12,34 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; -use std::sync::Arc; - use await_tree::InstrumentAwait; use itertools::Itertools; -use risingwave_common::util::tracing::TracingContext; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; use risingwave_hummock_sdk::LocalSstableInfo; -use risingwave_pb::stream_plan::barrier::BarrierKind; use risingwave_pb::stream_service::barrier_complete_response::GroupedSstableInfo; use risingwave_pb::stream_service::stream_service_server::StreamService; use risingwave_pb::stream_service::*; use risingwave_storage::dispatch_state_store; use risingwave_stream::error::StreamError; use risingwave_stream::executor::Barrier; -use risingwave_stream::task::{CollectResult, LocalStreamManager, StreamEnvironment}; +use risingwave_stream::task::{BarrierCompleteResult, LocalStreamManager, StreamEnvironment}; use thiserror_ext::AsReport; -use tonic::{Code, Request, Response, Status}; -use tracing::Instrument; +use tonic::{Request, Response, Status}; #[derive(Clone)] pub struct StreamServiceImpl { - mgr: Arc, + mgr: LocalStreamManager, env: StreamEnvironment, } impl StreamServiceImpl { - pub fn new(mgr: Arc, env: StreamEnvironment) -> Self { + pub fn new(mgr: LocalStreamManager, env: StreamEnvironment) -> Self { StreamServiceImpl { mgr, env } } } @@ -52,7 +46,7 @@ impl StreamService for StreamServiceImpl { request: Request, ) -> std::result::Result, Status> { let req = request.into_inner(); - let res = self.mgr.update_actors(&req.actors).await; + let res = self.mgr.update_actors(req.actors).await; match res { Err(e) => { error!(error = %e.as_report(), "failed to update stream actor"); @@ -70,10 +64,7 @@ impl StreamService for StreamServiceImpl { let req = request.into_inner(); let actor_id = req.actor_id; - let res = self - .mgr - .build_actors(actor_id.as_slice(), self.env.clone()) - .await; + let res = self.mgr.build_actors(actor_id).await; match res { Err(e) => { error!(error = %e.as_report(), "failed to build actors"); @@ -93,7 +84,7 @@ impl StreamService for StreamServiceImpl { ) -> std::result::Result, Status> { let req = request.into_inner(); - let res = self.mgr.update_actor_info(&req.info).await; + let res = self.mgr.update_actor_info(req.info).await; match res { Err(e) => { error!(error = %e.as_report(), "failed to update actor info table actor"); @@ -112,7 +103,7 @@ impl StreamService for StreamServiceImpl { ) -> std::result::Result, Status> { let req = request.into_inner(); let actors = req.actor_ids; - self.mgr.drop_actors(&actors).await?; + self.mgr.drop_actors(actors).await?; Ok(Response::new(DropActorsResponse { request_id: req.request_id, status: None, @@ -125,8 +116,7 @@ impl StreamService for StreamServiceImpl { request: Request, ) -> std::result::Result, Status> { let req = request.into_inner(); - self.mgr.stop_all_actors().await?; - self.env.dml_manager_ref().clear(); + self.mgr.reset(req.prev_epoch).await; Ok(Response::new(ForceStopActorsResponse { request_id: req.request_id, status: None, @@ -142,28 +132,6 @@ impl StreamService for StreamServiceImpl { let barrier = Barrier::from_protobuf(req.get_barrier().unwrap()).map_err(StreamError::from)?; - // The barrier might be outdated and been injected after recovery in some certain extreme - // scenarios. So some newly creating actors in the barrier are possibly not rebuilt during - // recovery. Check it here and return an error here if some actors are not found to - // avoid collection hang. We need some refine in meta side to remove this workaround since - // it will cause another round of unnecessary recovery. - let actor_ids = self.mgr.all_actor_ids().await; - let missing_actor_ids = req - .actor_ids_to_collect - .iter() - .filter(|id| !actor_ids.contains(id)) - .collect_vec(); - if !missing_actor_ids.is_empty() { - tracing::warn!( - "to collect actors not found, they should be cleaned when recovering: {:?}", - missing_actor_ids - ); - return Err(Status::new( - Code::InvalidArgument, - "to collect actors not found", - )); - } - self.mgr .send_barrier(barrier, req.actor_ids_to_send, req.actor_ids_to_collect) .await?; @@ -180,9 +148,9 @@ impl StreamService for StreamServiceImpl { request: Request, ) -> Result, Status> { let req = request.into_inner(); - let CollectResult { + let BarrierCompleteResult { create_mview_progress, - kind, + sync_result, } = self .mgr .collect_barrier(req.prev_epoch) @@ -192,39 +160,9 @@ impl StreamService for StreamServiceImpl { |err| tracing::error!(error = %err.as_report(), "failed to collect barrier"), )?; - let (synced_sstables, table_watermarks) = match kind { - BarrierKind::Unspecified => unreachable!(), - BarrierKind::Initial => { - if let Some(hummock) = self.env.state_store().as_hummock() { - let mce = hummock.get_pinned_version().max_committed_epoch(); - assert_eq!( - mce, req.prev_epoch, - "first epoch should match with the current version", - ); - } - tracing::info!( - epoch = req.prev_epoch, - "ignored syncing data for the first barrier" - ); - (Vec::new(), HashMap::new()) - } - BarrierKind::Barrier => (Vec::new(), HashMap::new()), - BarrierKind::Checkpoint => { - let span = TracingContext::from_protobuf(&req.tracing_context).attach( - tracing::info_span!("sync_epoch", prev_epoch = req.prev_epoch), - ); - - // Must finish syncing data written in the epoch before respond back to ensure - // persistence of the state. - let sync_result = self - .mgr - .sync_epoch(req.prev_epoch) - .instrument(span) - .instrument_await(format!("sync_epoch (epoch {})", req.prev_epoch)) - .await?; - (sync_result.uncommitted_ssts, sync_result.table_watermarks) - } - }; + let (synced_sstables, table_watermarks) = sync_result + .map(|sync_result| (sync_result.uncommitted_ssts, sync_result.table_watermarks)) + .unwrap_or_default(); Ok(Response::new(BarrierCompleteResponse { request_id: req.request_id, diff --git a/src/compute/src/server.rs b/src/compute/src/server.rs index 299e87adecf3e..fa15dcc3a38fe 100644 --- a/src/compute/src/server.rs +++ b/src/compute/src/server.rs @@ -28,6 +28,7 @@ use risingwave_common::config::{ }; use risingwave_common::monitor::connection::{RouterExt, TcpConfig}; use risingwave_common::system_param::local_manager::LocalSystemParamsManager; +use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_common::telemetry::manager::TelemetryManager; use risingwave_common::telemetry::telemetry_env_enabled; use risingwave_common::util::addr::HostAddr; @@ -38,6 +39,7 @@ use risingwave_common_service::metrics_manager::MetricsManager; use risingwave_common_service::observer_manager::ObserverManager; use risingwave_common_service::tracing::TracingExtractLayer; use risingwave_connector::source::monitor::GLOBAL_SOURCE_METRICS; +use risingwave_dml::dml_manager::DmlManager; use risingwave_pb::common::WorkerType; use risingwave_pb::compute::config_service_server::ConfigServiceServer; use risingwave_pb::connector_service::SinkPayloadFormat; @@ -48,7 +50,6 @@ use risingwave_pb::stream_service::stream_service_server::StreamServiceServer; use risingwave_pb::task_service::exchange_service_server::ExchangeServiceServer; use risingwave_pb::task_service::task_service_server::TaskServiceServer; use risingwave_rpc_client::{ComputeClientPool, ConnectorClient, ExtraInfoSourceRef, MetaClient}; -use risingwave_source::dml_manager::DmlManager; use risingwave_storage::hummock::compactor::{ start_compactor, CompactionExecutor, CompactorContext, }; @@ -271,13 +272,6 @@ pub async fn compute_node_serve( config.batch.clone(), batch_manager_metrics, )); - let stream_mgr = Arc::new(LocalStreamManager::new( - advertise_addr.clone(), - state_store.clone(), - streaming_metrics.clone(), - config.streaming.clone(), - await_tree_config.clone(), - )); // NOTE: Due to some limits, we use `compute_memory_bytes + storage_memory_bytes` as // `total_compute_memory_bytes` for memory control. This is just a workaround for some @@ -304,13 +298,6 @@ pub async fn compute_node_serve( // Run a background heap profiler heap_profiler.start(); - let watermark_epoch = memory_mgr.get_watermark_epoch(); - // Set back watermark epoch to stream mgr. Executor will read epoch from stream manager instead - // of lru manager. - stream_mgr.set_watermark_epoch(watermark_epoch).await; - - let grpc_await_tree_reg = await_tree_config - .map(|config| AwaitTreeRegistryRef::new(await_tree::Registry::new(config).into())); let dml_mgr = Arc::new(DmlManager::new( worker_id, config.streaming.developer.dml_channel_initial_permits, @@ -365,6 +352,16 @@ pub async fn compute_node_serve( meta_client.clone(), ); + let stream_mgr = LocalStreamManager::new( + stream_env.clone(), + streaming_metrics.clone(), + await_tree_config.clone(), + memory_mgr.get_watermark_epoch(), + ); + + let grpc_await_tree_reg = await_tree_config + .map(|config| AwaitTreeRegistryRef::new(await_tree::Registry::new(config).into())); + // Generally, one may use `risedev ctl trace` to manually get the trace reports. However, if // this is not the case, we can use the following command to get it printed into the logs // periodically. diff --git a/src/compute/tests/cdc_tests.rs b/src/compute/tests/cdc_tests.rs index 227948c853e76..760c76ca6deba 100644 --- a/src/compute/tests/cdc_tests.rs +++ b/src/compute/tests/cdc_tests.rs @@ -16,7 +16,6 @@ #![feature(coroutines)] use std::collections::{HashMap, HashSet}; -use std::marker::PhantomData; use std::str::FromStr; use std::sync::atomic::AtomicU64; use std::sync::Arc; @@ -35,7 +34,7 @@ use risingwave_connector::source::cdc::external::mock_external_table::MockExtern use risingwave_connector::source::cdc::external::{ DebeziumOffset, DebeziumSourceOffset, ExternalTableReaderImpl, MySqlOffset, SchemaTableName, }; -use risingwave_connector::source::cdc::{CdcSplitBase, DebeziumCdcSplit, MySqlCdcSplit}; +use risingwave_connector::source::cdc::DebeziumCdcSplit; use risingwave_connector::source::SplitImpl; use risingwave_hummock_sdk::to_committed_batch_query_epoch; use risingwave_storage::memory::MemoryStateStore; @@ -45,31 +44,22 @@ use risingwave_stream::error::StreamResult; use risingwave_stream::executor::monitor::StreamingMetrics; use risingwave_stream::executor::test_utils::MockSource; use risingwave_stream::executor::{ - expect_first_barrier, ActorContext, AddMutation, Barrier, BoxedExecutor as StreamBoxedExecutor, - BoxedMessageStream, CdcBackfillExecutor, Executor, ExecutorInfo, ExternalStorageTable, - MaterializeExecutor, Message, Mutation, PkIndices, PkIndicesRef, StreamExecutorError, + expect_first_barrier, ActorContext, AddMutation, Barrier, BoxedMessageStream, + CdcBackfillExecutor, Execute, Executor as StreamExecutor, ExecutorInfo, ExternalStorageTable, + MaterializeExecutor, Message, Mutation, StreamExecutorError, }; // mock upstream binlog offset starting from "1.binlog, pos=0" pub struct MockOffsetGenExecutor { - upstream: Option, - - schema: Schema, - - pk_indices: PkIndices, - - identity: String, + upstream: Option, start_offset: u32, } impl MockOffsetGenExecutor { - pub fn new(upstream: StreamBoxedExecutor, schema: Schema, pk_indices: PkIndices) -> Self { + pub fn new(upstream: StreamExecutor) -> Self { Self { upstream: Some(upstream), - schema, - pk_indices, - identity: "MockOffsetGenExecutor".to_string(), start_offset: 0, } } @@ -132,22 +122,10 @@ impl MockOffsetGenExecutor { } } -impl Executor for MockOffsetGenExecutor { +impl Execute for MockOffsetGenExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.pk_indices - } - - fn identity(&self) -> &str { - &self.identity - } } #[tokio::test] @@ -155,21 +133,26 @@ async fn test_cdc_backfill() -> StreamResult<()> { use risingwave_common::types::DataType; let memory_state_store = MemoryStateStore::new(); - let table_id = TableId::new(1002); - let schema = Schema::new(vec![ - Field::unnamed(DataType::Jsonb), // payload - Field::unnamed(DataType::Varchar), // _rw_offset - ]); - let column_ids = vec![0.into(), 1.into()]; - - let pk_indices = vec![0]; - - let (mut tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); - let _actor_ctx = ActorContext::create(0x3a3a3a); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor( + Schema::new(vec![ + Field::unnamed(DataType::Jsonb), // payload + ]), + vec![0], + ); // mock upstream offset (start from "1.binlog, pos=0") for ingested chunks - let mock_offset_executor = - MockOffsetGenExecutor::new(Box::new(source), schema.clone(), pk_indices.clone()); + let mock_offset_executor = StreamExecutor::new( + ExecutorInfo { + schema: Schema::new(vec![ + Field::unnamed(DataType::Jsonb), // payload + Field::unnamed(DataType::Varchar), // _rw_offset + ]), + pk_indices: vec![0], + identity: "MockOffsetGenExecutor".to_string(), + }, + MockOffsetGenExecutor::new(source).boxed(), + ); let binlog_file = String::from("1.binlog"); // mock binlog watermarks for backfill @@ -189,13 +172,15 @@ async fn test_cdc_backfill() -> StreamResult<()> { Field::with_name(DataType::Int64, "id"), // primary key Field::with_name(DataType::Float64, "price"), ]); + let table_pk_indices = vec![0]; + let table_pk_order_types = vec![OrderType::ascending()]; let external_table = ExternalStorageTable::new( - table_id, + TableId::new(1234), table_name, ExternalTableReaderImpl::Mock(MockExternalTableReader::new(binlog_watermarks)), table_schema.clone(), - vec![OrderType::ascending()], - pk_indices, + table_pk_order_types, + table_pk_indices.clone(), vec![0, 1], ); @@ -225,31 +210,35 @@ async fn test_cdc_backfill() -> StreamResult<()> { vec![0_usize], ) .await; - let info = ExecutorInfo { - schema: table_schema.clone(), - pk_indices: vec![0], - identity: "CdcBackfillExecutor".to_string(), - }; - let cdc_backfill = CdcBackfillExecutor::new( - ActorContext::create(actor_id), - info, - external_table, - Box::new(mock_offset_executor), - vec![0, 1], - None, - Arc::new(StreamingMetrics::unused()), - state_table, - 4, // 4 rows in a snapshot chunk + + let cdc_backfill = StreamExecutor::new( + ExecutorInfo { + schema: table_schema.clone(), + pk_indices: table_pk_indices, + identity: "CdcBackfillExecutor".to_string(), + }, + CdcBackfillExecutor::new( + ActorContext::for_test(actor_id), + external_table, + mock_offset_executor, + vec![0, 1], + None, + Arc::new(StreamingMetrics::unused()), + state_table, + 4, // 4 rows in a snapshot chunk + false, + ) + .boxed(), ); // Create a `MaterializeExecutor` to write the changes to storage. + let materialize_table_id = TableId::new(5678); let mut materialize = MaterializeExecutor::for_test( - Box::new(cdc_backfill), + cdc_backfill, memory_state_store.clone(), - table_id, + materialize_table_id, vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids.clone(), - 4, + vec![0.into(), 1.into()], Arc::new(AtomicU64::new(0)), ConflictBehavior::Overwrite, ) @@ -295,17 +284,7 @@ async fn test_cdc_backfill() -> StreamResult<()> { let mut splits = HashMap::new(); splits.insert( actor_id, - vec![SplitImpl::MysqlCdc(DebeziumCdcSplit { - mysql_split: Some(MySqlCdcSplit { - inner: CdcSplitBase { - split_id: 0, - start_offset: None, - snapshot_done: false, - }, - }), - pg_split: None, - _phantom: PhantomData, - })], + vec![SplitImpl::MysqlCdc(DebeziumCdcSplit::new(0, None, None))], ); let init_barrier = Barrier::new_test_barrier(curr_epoch).with_mutation(Mutation::Add(AddMutation { @@ -356,7 +335,7 @@ async fn test_cdc_backfill() -> StreamResult<()> { // Since we have not polled `Materialize`, we cannot scan anything from this table let table = StorageTable::for_test( memory_state_store.clone(), - table_id, + materialize_table_id, column_descs.clone(), vec![OrderType::ascending()], vec![0], diff --git a/src/compute/tests/integration_tests.rs b/src/compute/tests/integration_tests.rs index 71e3c564ef758..210559c64ff15 100644 --- a/src/compute/tests/integration_tests.rs +++ b/src/compute/tests/integration_tests.rs @@ -32,7 +32,6 @@ use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::{ ColumnDesc, ColumnId, ConflictBehavior, Field, Schema, TableId, INITIAL_TABLE_VERSION_ID, }; -use risingwave_common::error::{Result, RwError}; use risingwave_common::row::OwnedRow; use risingwave_common::system_param::local_manager::LocalSystemParamsManager; use risingwave_common::test_prelude::DataChunkTestExt; @@ -40,13 +39,13 @@ use risingwave_common::types::{DataType, IntoOrdered}; use risingwave_common::util::epoch::{test_epoch, EpochExt, EpochPair}; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::sort_util::{ColumnOrder, OrderType}; +use risingwave_connector::source::reader::desc::test_utils::create_source_desc_builder; use risingwave_connector::source::SourceCtrlOpts; use risingwave_connector::ConnectorParams; +use risingwave_dml::dml_manager::DmlManager; use risingwave_hummock_sdk::to_committed_batch_query_epoch; use risingwave_pb::catalog::StreamSourceInfo; use risingwave_pb::plan_common::PbRowFormatType; -use risingwave_source::connector_test_utils::create_source_desc_builder; -use risingwave_source::dml_manager::DmlManager; use risingwave_storage::memory::MemoryStateStore; use risingwave_storage::panic_store::PanicStateStore; use risingwave_storage::table::batch_table::storage_table::StorageTable; @@ -57,7 +56,7 @@ use risingwave_stream::executor::monitor::StreamingMetrics; use risingwave_stream::executor::row_id_gen::RowIdGenExecutor; use risingwave_stream::executor::source_executor::SourceExecutor; use risingwave_stream::executor::{ - ActorContext, Barrier, Executor, ExecutorInfo, MaterializeExecutor, Message, PkIndices, + ActorContext, Barrier, Execute, Executor, ExecutorInfo, MaterializeExecutor, Message, PkIndices, }; use tokio::sync::mpsc::unbounded_channel; @@ -159,60 +158,62 @@ async fn test_table_materialize() -> StreamResult<()> { let barrier_tx = Arc::new(barrier_tx); let vnodes = Bitmap::from_bytes(&[0b11111111]); - let actor_ctx = ActorContext::create(0x3f3f3f); + let actor_ctx = ActorContext::for_test(0x3f3f3f); let system_params_manager = LocalSystemParamsManager::for_test(); // Create a `SourceExecutor` to read the changes. - let source_executor = SourceExecutor::::new( - actor_ctx.clone(), + let source_executor = Executor::new( ExecutorInfo { schema: all_schema.clone(), pk_indices: pk_indices.clone(), identity: format!("SourceExecutor {:X}", 1), }, - None, // There is no external stream source. - Arc::new(StreamingMetrics::unused()), - barrier_rx, - system_params_manager.get_params(), - SourceCtrlOpts::default(), - ConnectorParams::default(), + SourceExecutor::::new( + actor_ctx.clone(), + None, // There is no external stream source. + Arc::new(StreamingMetrics::unused()), + barrier_rx, + system_params_manager.get_params(), + SourceCtrlOpts::default(), + ConnectorParams::default(), + ) + .boxed(), ); // Create a `DmlExecutor` to accept data change from users. - let dml_executor = DmlExecutor::new( + let dml_executor = Executor::new( ExecutorInfo { schema: all_schema.clone(), pk_indices: pk_indices.clone(), identity: format!("DmlExecutor {:X}", 2), }, - Box::new(source_executor), - dml_manager.clone(), - table_id, - INITIAL_TABLE_VERSION_ID, - column_descs.clone(), - 1024, + DmlExecutor::new( + source_executor, + dml_manager.clone(), + table_id, + INITIAL_TABLE_VERSION_ID, + column_descs.clone(), + 1024, + ) + .boxed(), ); - let row_id_gen_executor = RowIdGenExecutor::new( - actor_ctx, + let row_id_gen_executor = Executor::new( ExecutorInfo { schema: all_schema.clone(), pk_indices: pk_indices.clone(), identity: format!("RowIdGenExecutor {:X}", 3), }, - Box::new(dml_executor), - row_id_index, - vnodes, + RowIdGenExecutor::new(actor_ctx, dml_executor, row_id_index, vnodes).boxed(), ); // Create a `MaterializeExecutor` to write the changes to storage. let mut materialize = MaterializeExecutor::for_test( - Box::new(row_id_gen_executor), + row_id_gen_executor, memory_state_store.clone(), table_id, vec![ColumnOrder::new(0, OrderType::ascending())], all_column_ids.clone(), - 4, Arc::new(AtomicU64::new(0)), ConflictBehavior::NoCheck, ) @@ -297,7 +298,7 @@ async fn test_table_materialize() -> StreamResult<()> { barrier_tx_clone .send(Barrier::new_test_barrier(curr_epoch)) .unwrap(); - Ok::<_, RwError>(()) + anyhow::Ok(()) }); // Poll `Materialize`, should output the same insertion stream chunk. @@ -379,7 +380,7 @@ async fn test_table_materialize() -> StreamResult<()> { barrier_tx_clone .send(Barrier::new_test_barrier(curr_epoch)) .unwrap(); - Ok::<_, RwError>(()) + anyhow::Ok(()) }); // Poll `Materialize`, should output the same deletion stream chunk. @@ -429,7 +430,7 @@ async fn test_table_materialize() -> StreamResult<()> { } #[tokio::test] -async fn test_row_seq_scan() -> Result<()> { +async fn test_row_seq_scan() -> StreamResult<()> { // In this test we test if the memtable can be correctly scanned for K-V pair insertions. let memory_state_store = MemoryStateStore::new(); diff --git a/src/config/ci-delete-range-test.toml b/src/config/ci-delete-range-test.toml deleted file mode 100644 index 33313ca092c4d..0000000000000 --- a/src/config/ci-delete-range-test.toml +++ /dev/null @@ -1,8 +0,0 @@ -[meta] -vacuum_interval_sec = 10 - -[server] -telemetry_enabled = false - -[system] -max_concurrent_creating_streaming_jobs = 0 \ No newline at end of file diff --git a/src/config/ci-meta-backup-test.toml b/src/config/ci-meta-backup-test.toml index 9559390360e33..c15230262655c 100644 --- a/src/config/ci-meta-backup-test.toml +++ b/src/config/ci-meta-backup-test.toml @@ -4,7 +4,6 @@ collect_gc_watermark_spin_interval_sec = 1 vacuum_interval_sec = 10 [system] -backup_storage_url = "minio://hummockadmin:hummockadmin@127.0.0.1:9301/hummock001" backup_storage_directory = "backup" [server] diff --git a/src/config/ci.toml b/src/config/ci.toml index 79a0dbca06f23..db207ebf44412 100644 --- a/src/config/ci.toml +++ b/src/config/ci.toml @@ -2,6 +2,11 @@ disable_recovery = true max_heartbeat_interval_secs = 600 +[meta.compaction_config] +level0_tier_compact_file_number = 6 +level0_overlapping_sub_level_compact_level_count = 3 +level0_max_compact_file_number = 96 + [streaming] in_flight_barrier_nums = 10 @@ -14,4 +19,4 @@ imm_merge_threshold = 2 [system] barrier_interval_ms = 250 checkpoint_frequency = 5 -max_concurrent_creating_streaming_jobs = 0 +max_concurrent_creating_streaming_jobs = 0 \ No newline at end of file diff --git a/src/config/docs.md b/src/config/docs.md new file mode 100644 index 0000000000000..cf28f0ca9e2c6 --- /dev/null +++ b/src/config/docs.md @@ -0,0 +1,155 @@ +# RisingWave System Configurations + +This page is automatically generated by `./risedev generate-example-config` + +## batch + +| Config | Description | Default | +|--------|-------------|---------| +| distributed_query_limit | | | +| enable_barrier_read | | false | +| frontend_compute_runtime_worker_threads | frontend compute runtime worker threads | 4 | +| statement_timeout_in_sec | Timeout for a batch query in seconds. | 3600 | +| worker_threads_num | The thread number of the batch task runtime in the compute node. The default value is decided by `tokio`. | | + +## meta + +| Config | Description | Default | +|--------|-------------|---------| +| backend | | "Mem" | +| collect_gc_watermark_spin_interval_sec | The spin interval when collecting global GC watermark in hummock. | 5 | +| compaction_task_max_heartbeat_interval_secs | | 30 | +| compaction_task_max_progress_interval_secs | | 600 | +| cut_table_size_limit | | 1073741824 | +| dangerous_max_idle_secs | After specified seconds of idle (no mview or flush), the process will be exited. It is mainly useful for playgrounds. | | +| default_parallelism | The default global parallelism for all streaming jobs, if user doesn't specify the parallelism, this value will be used. `FULL` means use all available parallelism units, otherwise it's a number. | "Full" | +| disable_automatic_parallelism_control | Whether to disable adaptive-scaling feature. | false | +| disable_recovery | Whether to enable fail-on-recovery. Should only be used in e2e tests. | false | +| do_not_config_object_storage_lifecycle | Whether config object storage bucket lifecycle to purge stale data. | false | +| enable_committed_sst_sanity_check | Enable sanity check when SSTs are committed. | false | +| enable_compaction_deterministic | Whether to enable deterministic compaction scheduling, which will disable all auto scheduling of compaction tasks. Should only be used in e2e tests. | false | +| enable_hummock_data_archive | If enabled, SSTable object file and version delta will be retained. SSTable object file need to be deleted via full GC. version delta need to be manually deleted. | false | +| event_log_channel_max_size | Keeps the latest N events per channel. | 10 | +| event_log_enabled | | true | +| full_gc_interval_sec | Interval of automatic hummock full GC. | 86400 | +| hummock_version_checkpoint_interval_sec | Interval of hummock version checkpoint. | 30 | +| hybird_partition_vnode_count | | 4 | +| max_heartbeat_interval_secs | Maximum allowed heartbeat interval in seconds. | 300 | +| meta_leader_lease_secs | | 30 | +| min_delta_log_num_for_hummock_version_checkpoint | The minimum delta log number a new checkpoint should compact, otherwise the checkpoint attempt is rejected. | 10 | +| min_sst_retention_time_sec | Objects within `min_sst_retention_time_sec` won't be deleted by hummock full GC, even they are dangling. | 86400 | +| min_table_split_write_throughput | If the size of one table is smaller than `min_table_split_write_throughput`, we would not split it to an single group. | 4194304 | +| move_table_size_limit | | 10737418240 | +| node_num_monitor_interval_sec | | 10 | +| parallelism_control_batch_size | The number of streaming jobs per scaling operation. | 10 | +| parallelism_control_trigger_first_delay_sec | The first delay of parallelism control. | 30 | +| parallelism_control_trigger_period_sec | The period of parallelism control trigger. | 10 | +| partition_vnode_count | | 16 | +| periodic_compaction_interval_sec | Schedule compaction for all compaction groups with this interval. | 60 | +| periodic_space_reclaim_compaction_interval_sec | Schedule space_reclaim compaction for all compaction groups with this interval. | 3600 | +| periodic_split_compact_group_interval_sec | | 10 | +| periodic_tombstone_reclaim_compaction_interval_sec | | 600 | +| periodic_ttl_reclaim_compaction_interval_sec | Schedule ttl_reclaim compaction for all compaction groups with this interval. | 1800 | +| split_group_size_limit | | 68719476736 | +| table_write_throughput_threshold | | 16777216 | +| unrecognized | | | +| vacuum_interval_sec | Interval of invoking a vacuum job, to remove stale metadata from meta store and objects from object store. | 30 | +| vacuum_spin_interval_ms | The spin interval inside a vacuum job. It avoids the vacuum job monopolizing resources of meta node. | 10 | + +## meta.compaction_config + +| Config | Description | Default | +|--------|-------------|---------| +| compaction_filter_mask | | 6 | +| enable_emergency_picker | | true | +| level0_max_compact_file_number | | 100 | +| level0_overlapping_sub_level_compact_level_count | | 12 | +| level0_stop_write_threshold_sub_level_number | | 300 | +| level0_sub_level_compact_level_count | | 3 | +| level0_tier_compact_file_number | | 12 | +| max_bytes_for_level_base | | 536870912 | +| max_bytes_for_level_multiplier | | 5 | +| max_compaction_bytes | | 2147483648 | +| max_space_reclaim_bytes | | 536870912 | +| max_sub_compaction | | 4 | +| sub_level_max_compaction_bytes | | 134217728 | +| target_file_size_base | | 33554432 | +| tombstone_reclaim_ratio | | 40 | + +## server + +| Config | Description | Default | +|--------|-------------|---------| +| connection_pool_size | | 16 | +| grpc_max_reset_stream | | 200 | +| heap_profiling | Enable heap profile dump when memory usage is high. | | +| heartbeat_interval_ms | The interval for periodic heartbeat from worker to the meta service. | 1000 | +| metrics_level | Used for control the metrics level, similar to log level. | "Info" | +| telemetry_enabled | | true | + +## storage + +| Config | Description | Default | +|--------|-------------|---------| +| block_cache_capacity_mb | Capacity of sstable block cache. | | +| cache_refill | | | +| check_compaction_result | | false | +| compact_iter_recreate_timeout_ms | | 600000 | +| compactor_fast_max_compact_delete_ratio | | 40 | +| compactor_fast_max_compact_task_size | | 2147483648 | +| compactor_max_sst_key_count | | 2097152 | +| compactor_max_sst_size | | 536870912 | +| compactor_max_task_multiplier | Compactor calculates the maximum number of tasks that can be executed on the node based on worker_num and compactor_max_task_multiplier. max_pull_task_count = worker_num * compactor_max_task_multiplier | 2.5 | +| compactor_memory_available_proportion | The percentage of memory available when compactor is deployed separately. non_reserved_memory_bytes = system_memory_available_bytes * compactor_memory_available_proportion | 0.8 | +| compactor_memory_limit_mb | | | +| data_file_cache | | | +| disable_remote_compactor | | false | +| enable_fast_compaction | | false | +| high_priority_ratio_in_percent | | | +| imm_merge_threshold | The threshold for the number of immutable memtables to merge to a new imm. | 0 | +| max_concurrent_compaction_task_number | | 16 | +| max_prefetch_block_number | max prefetch block number | 16 | +| max_preload_io_retry_times | | 3 | +| max_preload_wait_time_mill | | 0 | +| max_sub_compaction | Max sub compaction task numbers | 4 | +| max_version_pinning_duration_sec | | 10800 | +| mem_table_spill_threshold | The spill threshold for mem table. | 4194304 | +| meta_cache_capacity_mb | Capacity of sstable meta cache. | | +| meta_file_cache | | | +| min_sst_size_for_streaming_upload | Whether to enable streaming upload for sstable. | 33554432 | +| object_store | | | +| prefetch_buffer_capacity_mb | max memory usage for large query | | +| share_buffer_compaction_worker_threads_number | Worker threads number of dedicated tokio runtime for share buffer compaction. 0 means use tokio's default value (number of CPU core). | 4 | +| share_buffer_upload_concurrency | Number of tasks shared buffer can upload in parallel. | 8 | +| share_buffers_sync_parallelism | parallelism while syncing share buffers into L0 SST. Should NOT be 0. | 1 | +| shared_buffer_capacity_mb | Maximum shared buffer size, writes attempting to exceed the capacity will stall until there is enough space. | | +| shared_buffer_flush_ratio | The shared buffer will start flushing data to object when the ratio of memory usage to the shared buffer capacity exceed such ratio. | 0.800000011920929 | +| sstable_id_remote_fetch_number | Number of SST ids fetched from meta per RPC | 10 | +| write_conflict_detection_enabled | Whether to enable write conflict detection | true | + +## streaming + +| Config | Description | Default | +|--------|-------------|---------| +| actor_runtime_worker_threads_num | The thread number of the streaming actor runtime in the compute node. The default value is decided by `tokio`. | | +| async_stack_trace | Enable async stack tracing through `await-tree` for risectl. | "ReleaseVerbose" | +| in_flight_barrier_nums | The maximum number of barriers in-flight in the compute nodes. | 10000 | +| unique_user_stream_errors | Max unique user stream errors per actor | 10 | + +## system + +| Config | Description | Default | +|--------|-------------|---------| +| backup_storage_directory | Remote directory for storing snapshots. | | +| backup_storage_url | Remote storage url for storing snapshots. | | +| barrier_interval_ms | The interval of periodic barrier. | 1000 | +| block_size_kb | Size of each block in bytes in SST. | 64 | +| bloom_false_positive | False positive probability of bloom filter. | 0.001 | +| checkpoint_frequency | There will be a checkpoint for every n barriers. | 1 | +| data_directory | Remote directory for storing data and metadata objects. | | +| enable_tracing | Whether to enable distributed tracing. | false | +| max_concurrent_creating_streaming_jobs | Max number of concurrent creating streaming jobs. | 1 | +| parallel_compact_size_mb | | 512 | +| pause_on_next_bootstrap | Whether to pause all data sources on next bootstrap. | false | +| sstable_size_mb | Target size of the Sstable. | 256 | +| state_store | | | diff --git a/src/config/example.toml b/src/config/example.toml index 21d13f81fbdcd..e322a180f978e 100644 --- a/src/config/example.toml +++ b/src/config/example.toml @@ -21,11 +21,14 @@ periodic_compaction_interval_sec = 60 vacuum_interval_sec = 30 vacuum_spin_interval_ms = 10 hummock_version_checkpoint_interval_sec = 30 +enable_hummock_data_archive = false min_delta_log_num_for_hummock_version_checkpoint = 10 max_heartbeat_interval_secs = 300 disable_recovery = false -enable_scale_in_when_recovery = false -enable_automatic_parallelism_control = false +disable_automatic_parallelism_control = false +parallelism_control_batch_size = 10 +parallelism_control_trigger_period_sec = 10 +parallelism_control_trigger_first_delay_sec = 30 meta_leader_lease_secs = 30 default_parallelism = "Full" enable_compaction_deterministic = false @@ -54,25 +57,28 @@ max_bytes_for_level_base = 536870912 max_bytes_for_level_multiplier = 5 max_compaction_bytes = 2147483648 sub_level_max_compaction_bytes = 134217728 -level0_tier_compact_file_number = 6 +level0_tier_compact_file_number = 12 target_file_size_base = 33554432 compaction_filter_mask = 6 max_sub_compaction = 4 level0_stop_write_threshold_sub_level_number = 300 level0_sub_level_compact_level_count = 3 -level0_overlapping_sub_level_compact_level_count = 6 +level0_overlapping_sub_level_compact_level_count = 12 max_space_reclaim_bytes = 536870912 -level0_max_compact_file_number = 96 +level0_max_compact_file_number = 100 tombstone_reclaim_ratio = 40 enable_emergency_picker = true [meta.developer] meta_cached_traces_num = 256 meta_cached_traces_memory_limit_bytes = 134217728 +meta_enable_trivial_move = true +meta_enable_check_task_level_overlap = false [batch] enable_barrier_read = false statement_timeout_in_sec = 3600 +frontend_compute_runtime_worker_threads = 4 [batch.developer] batch_connector_message_buffer_size = 16 @@ -117,11 +123,11 @@ compactor_max_sst_key_count = 2097152 compact_iter_recreate_timeout_ms = 600000 compactor_max_sst_size = 536870912 enable_fast_compaction = false -check_fast_compaction_result = false +check_compaction_result = false max_preload_io_retry_times = 3 compactor_fast_max_compact_delete_ratio = 40 compactor_fast_max_compact_task_size = 2147483648 -mem_table_spill_threshold = 0 +mem_table_spill_threshold = 4194304 [storage.data_file_cache] dir = "" @@ -177,6 +183,11 @@ object_store_nodelay = true object_store_req_retry_interval_ms = 20 object_store_req_retry_max_delay_ms = 10000 object_store_req_retry_max_attempts = 8 +retry_unknown_service_error = false + +[storage.object_store.s3.developer] +object_store_retry_unknown_service_error = false +object_store_retryable_service_error_codes = ["SlowDown", "TooManyRequests"] [system] barrier_interval_ms = 1000 @@ -185,9 +196,6 @@ sstable_size_mb = 256 parallel_compact_size_mb = 512 block_size_kb = 64 bloom_false_positive = 0.001 -backup_storage_url = "memory" -backup_storage_directory = "backup" max_concurrent_creating_streaming_jobs = 1 pause_on_next_bootstrap = false -wasm_storage_url = "fs://.risingwave/data" enable_tracing = false diff --git a/src/connector/Cargo.toml b/src/connector/Cargo.toml index 791cc076d12e2..1efdce6a2d875 100644 --- a/src/connector/Cargo.toml +++ b/src/connector/Cargo.toml @@ -22,7 +22,9 @@ apache-avro = { git = "https://github.com/risingwavelabs/avro", rev = "d0846a16c "xz", ] } arrow-array = { workspace = true } +arrow-row = { workspace = true } arrow-schema = { workspace = true } +arrow-select = { workspace = true } async-nats = "0.33" async-trait = "0.1" auto_enums = { version = "0.8", features = ["futures03"] } @@ -36,7 +38,7 @@ aws-smithy-runtime-api = { workspace = true } aws-smithy-types = { workspace = true } aws-smithy-types-convert = { version = "0.60.1", features = ["convert-chrono"] } aws-types = { workspace = true } -base64 = "0.21" +base64 = "0.22" byteorder = "1" bytes = { version = "1", features = ["serde"] } chrono = { version = "0.4", default-features = false, features = [ @@ -55,7 +57,7 @@ futures = { version = "0.3", default-features = false, features = ["alloc"] } futures-async-stream = { workspace = true } gcp-bigquery-client = "0.18.0" glob = "0.3" -google-cloud-pubsub = "0.22" +google-cloud-pubsub = "0.23" http = "0.2" hyper = { version = "0.14", features = [ "client", @@ -85,13 +87,15 @@ parking_lot = "0.12" paste = "1" prometheus = { version = "0.13", features = ["process"] } prost = { version = "0.12", features = ["no-recursion-limit"] } -prost-reflect = "0.12" +prost-reflect = "0.13" prost-types = "0.12" protobuf-native = "0.2.1" pulsar = { version = "6.1", default-features = false, features = [ "tokio-runtime", "telemetry", "auth-oauth2", + "lz4", + "zstd", ] } rdkafka = { workspace = true, features = [ "cmake-build", @@ -110,15 +114,17 @@ risingwave_jni_core = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } rust_decimal = "1" +rw_futures_util = { workspace = true } serde = { version = "1", features = ["derive", "rc"] } serde_derive = "1" serde_json = "1" serde_with = { version = "3", features = ["json"] } simd-json = "0.13.3" strum = "0.25" -strum_macros = "0.25" +strum_macros = "0.26" tempfile = "3" thiserror = "1" +thiserror-ext = { workspace = true } time = "0.3.30" tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", @@ -129,14 +135,15 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [ "signal", "fs", ] } -tokio-postgres = "0.7" +tokio-postgres = { version = "0.7", features = ["with-uuid-1"] } tokio-retry = "0.3" tokio-stream = "0.1" tokio-util = { version = "0.7", features = ["codec", "io"] } -tonic = "0.10.2" +tonic = { workspace = true } tracing = "0.1" url = "2" urlencoding = "2" +uuid = { version = "1", features = ["v4", "fast-rng"] } with_options = { path = "./with_options" } yup-oauth2 = "8.3" @@ -144,9 +151,11 @@ yup-oauth2 = "8.3" workspace-hack = { path = "../workspace-hack" } [dev-dependencies] +assert_matches = "1" criterion = { workspace = true, features = ["async_tokio", "async"] } deltalake = { workspace = true, features = ["datafusion"] } expect-test = "1" +paste = "1" pretty_assertions = "1" quote = "1" rand = "0.8" @@ -170,5 +179,9 @@ harness = false name = "nexmark_integration" harness = false +[[bench]] +name = "json_parser" +harness = false + [lints] workspace = true diff --git a/src/source/benches/json_parser.rs b/src/connector/benches/json_parser.rs similarity index 100% rename from src/source/benches/json_parser.rs rename to src/connector/benches/json_parser.rs diff --git a/src/connector/benches/nexmark_integration.rs b/src/connector/benches/nexmark_integration.rs index 951d42e594732..e6388ed4b0d25 100644 --- a/src/connector/benches/nexmark_integration.rs +++ b/src/connector/benches/nexmark_integration.rs @@ -19,14 +19,14 @@ use std::sync::LazyLock; use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; use futures::{FutureExt, StreamExt, TryStreamExt}; use itertools::Itertools; +use risingwave_common::array::StreamChunk; use risingwave_common::catalog::ColumnId; use risingwave_common::types::DataType; use risingwave_connector::parser::{ ByteStreamSourceParser, JsonParser, SourceParserIntoStreamExt, SpecificParserConfig, }; use risingwave_connector::source::{ - BoxSourceStream, BoxSourceWithStateStream, SourceColumnDesc, SourceMessage, SourceMeta, - StreamChunkWithState, + BoxChunkSourceStream, BoxSourceStream, SourceColumnDesc, SourceMessage, SourceMeta, }; use tracing::Level; use tracing_subscriber::prelude::*; @@ -90,9 +90,8 @@ fn make_parser() -> impl ByteStreamSourceParser { JsonParser::new(props, columns, Default::default()).unwrap() } -fn make_stream_iter() -> impl Iterator { - let mut stream: BoxSourceWithStateStream = - make_parser().into_stream(make_data_stream()).boxed(); +fn make_stream_iter() -> impl Iterator { + let mut stream: BoxChunkSourceStream = make_parser().into_stream(make_data_stream()).boxed(); std::iter::from_fn(move || { stream diff --git a/src/connector/src/aws_utils.rs b/src/connector/src/aws_utils.rs index 4f9b1bd08ae57..1578c7b844422 100644 --- a/src/connector/src/aws_utils.rs +++ b/src/connector/src/aws_utils.rs @@ -15,14 +15,13 @@ use std::collections::HashMap; use std::time::Duration; +use anyhow::Context; use aws_config::timeout::TimeoutConfig; -use aws_sdk_s3::error::DisplayErrorContext; use aws_sdk_s3::{client as s3_client, config as s3_config}; -use risingwave_common::error::ErrorCode::InternalError; -use risingwave_common::error::{Result, RwError}; use url::Url; use crate::common::AwsAuthProps; +use crate::error::ConnectorResult; const AWS_CUSTOM_CONFIG_KEY: [&str; 3] = ["retry_times", "conn_timeout", "read_timeout"]; @@ -108,10 +107,10 @@ pub fn s3_client( pub async fn load_file_descriptor_from_s3( location: &Url, config: &AwsAuthProps, -) -> Result> { +) -> ConnectorResult> { let bucket = location .domain() - .ok_or_else(|| RwError::from(InternalError(format!("Illegal file path {}", location))))?; + .with_context(|| format!("illegal file path {}", location))?; let key = location.path().replace('/', ""); let sdk_config = config.build_config().await?; let s3_client = s3_client(&sdk_config, Some(default_conn_config())); @@ -121,18 +120,12 @@ pub async fn load_file_descriptor_from_s3( .key(&key) .send() .await - .map_err(|e| { - RwError::from(InternalError(format!( - "get file {} err:{}", - location, - DisplayErrorContext(e) - ))) - })?; + .with_context(|| format!("failed to get file from s3 at `{}`", location))?; let body = response .body .collect() .await - .map_err(|e| RwError::from(InternalError(format!("Read file from s3 {}", e))))?; + .with_context(|| format!("failed to read file from s3 at `{}`", location))?; Ok(body.into_bytes().to_vec()) } diff --git a/src/connector/src/common.rs b/src/connector/src/common.rs index df61dc157b40b..d5944eb07fa3c 100644 --- a/src/connector/src/common.rs +++ b/src/connector/src/common.rs @@ -17,15 +17,14 @@ use std::collections::HashMap; use std::io::Write; use std::time::Duration; -use anyhow::{anyhow, Ok}; +use anyhow::{anyhow, Context}; use async_nats::jetstream::consumer::DeliverPolicy; use async_nats::jetstream::{self}; use aws_sdk_kinesis::Client as KinesisClient; use pulsar::authentication::oauth2::{OAuth2Authentication, OAuth2Params}; use pulsar::{Authentication, Pulsar, TokioExecutor}; use rdkafka::ClientConfig; -use risingwave_common::error::ErrorCode::InvalidParameterValue; -use risingwave_common::error::{anyhow_error, RwError}; +use risingwave_common::bail; use serde_derive::Deserialize; use serde_with::json::JsonString; use serde_with::{serde_as, DisplayFromStr}; @@ -36,12 +35,13 @@ use with_options::WithOptions; use crate::aws_utils::load_file_descriptor_from_s3; use crate::deserialize_duration_from_string; +use crate::error::ConnectorResult; use crate::sink::SinkError; use crate::source::nats::source::NatsOffset; // The file describes the common abstractions for each connector and can be used in both source and // sink. -pub const BROKER_REWRITE_MAP_KEY: &str = "broker.rewrite.endpoints"; +pub const PRIVATE_LINK_BROKER_REWRITE_MAP_KEY: &str = "broker.rewrite.endpoints"; pub const PRIVATE_LINK_TARGETS_KEY: &str = "privatelink.targets"; #[derive(Debug, Clone, Deserialize)] @@ -73,7 +73,7 @@ pub struct AwsAuthProps { } impl AwsAuthProps { - async fn build_region(&self) -> anyhow::Result { + async fn build_region(&self) -> ConnectorResult { if let Some(region_name) = &self.region { Ok(Region::new(region_name.clone())) } else { @@ -86,11 +86,11 @@ impl AwsAuthProps { .build() .region() .await - .ok_or_else(|| anyhow::format_err!("region should be provided"))?) + .context("region should be provided")?) } } - fn build_credential_provider(&self) -> anyhow::Result { + fn build_credential_provider(&self) -> ConnectorResult { if self.access_key.is_some() && self.secret_key.is_some() { Ok(SharedCredentialsProvider::new( aws_credential_types::Credentials::from_keys( @@ -100,16 +100,14 @@ impl AwsAuthProps { ), )) } else { - Err(anyhow!( - "Both \"access_key\" and \"secret_access\" are required." - )) + bail!("Both \"access_key\" and \"secret_access\" are required.") } } async fn with_role_provider( &self, credential: SharedCredentialsProvider, - ) -> anyhow::Result { + ) -> ConnectorResult { if let Some(role_name) = &self.arn { let region = self.build_region().await?; let mut role = AssumeRoleProvider::builder(role_name) @@ -125,7 +123,7 @@ impl AwsAuthProps { } } - pub async fn build_config(&self) -> anyhow::Result { + pub async fn build_config(&self) -> ConnectorResult { let region = self.build_region().await?; let credentials_provider = self .with_role_provider(self.build_credential_provider()?) @@ -148,10 +146,6 @@ pub struct KafkaCommon { #[serde(rename = "properties.bootstrap.server", alias = "kafka.brokers")] pub brokers: String, - #[serde(rename = "broker.rewrite.endpoints")] - #[serde_as(as = "Option")] - pub broker_rewrite_map: Option>, - #[serde(rename = "topic", alias = "kafka.topic")] pub topic: String, @@ -224,6 +218,15 @@ pub struct KafkaCommon { sasl_oathbearer_config: Option, } +#[serde_as] +#[derive(Debug, Clone, Deserialize, WithOptions)] +pub struct KafkaPrivateLinkCommon { + /// This is generated from `private_link_targets` and `private_link_endpoint` in frontend, instead of given by users. + #[serde(rename = "broker.rewrite.endpoints")] + #[serde_as(as = "Option")] + pub broker_rewrite_map: Option>, +} + const fn default_kafka_sync_call_timeout() -> Duration { Duration::from_secs(5) } @@ -255,6 +258,10 @@ pub struct RdKafkaPropertiesCommon { #[serde(rename = "properties.client.id")] #[serde_as(as = "Option")] pub client_id: Option, + + #[serde(rename = "properties.enable.ssl.certificate.verification")] + #[serde_as(as = "Option")] + pub enable_ssl_certificate_verification: Option, } impl RdKafkaPropertiesCommon { @@ -271,6 +278,9 @@ impl RdKafkaPropertiesCommon { if let Some(v) = self.client_id.as_ref() { c.set("client.id", v); } + if let Some(v) = self.enable_ssl_certificate_verification { + c.set("enable.ssl.certificate.verification", v.to_string()); + } } } @@ -375,12 +385,19 @@ pub struct PulsarOauthCommon { pub scope: Option, } +fn create_credential_temp_file(credentials: &[u8]) -> std::io::Result { + let mut f = NamedTempFile::new()?; + f.write_all(credentials)?; + f.as_file().sync_all()?; + Ok(f) +} + impl PulsarCommon { pub(crate) async fn build_client( &self, oauth: &Option, aws_auth_props: &AwsAuthProps, - ) -> anyhow::Result> { + ) -> ConnectorResult> { let mut pulsar_builder = Pulsar::builder(&self.service_url, TokioExecutor); let mut temp_file = None; if let Some(oauth) = oauth.as_ref() { @@ -388,17 +405,14 @@ impl PulsarCommon { match url.scheme() { "s3" => { let credentials = load_file_descriptor_from_s3(&url, aws_auth_props).await?; - let mut f = NamedTempFile::new()?; - f.write_all(&credentials)?; - f.as_file().sync_all()?; - temp_file = Some(f); + temp_file = Some( + create_credential_temp_file(&credentials) + .context("failed to create temp file for pulsar credentials")?, + ); } "file" => {} _ => { - return Err(RwError::from(InvalidParameterValue(String::from( - "invalid credentials_url, only file url and s3 url are supported", - ))) - .into()); + bail!("invalid credentials_url, only file url and s3 url are supported",); } } @@ -469,7 +483,7 @@ pub struct KinesisCommon { } impl KinesisCommon { - pub(crate) async fn build_client(&self) -> anyhow::Result { + pub(crate) async fn build_client(&self) -> ConnectorResult { let config = AwsAuthProps { region: Some(self.stream_region.clone()), endpoint: self.endpoint.clone(), @@ -531,7 +545,7 @@ pub struct NatsCommon { } impl NatsCommon { - pub(crate) async fn build_client(&self) -> anyhow::Result { + pub(crate) async fn build_client(&self) -> ConnectorResult { let mut connect_options = async_nats::ConnectOptions::new(); match self.connect_mode.as_str() { "user_and_password" => { @@ -541,9 +555,7 @@ impl NatsCommon { connect_options = connect_options.user_and_password(v_user.into(), v_password.into()) } else { - return Err(anyhow_error!( - "nats connect mode is user_and_password, but user or password is empty" - )); + bail!("nats connect mode is user_and_password, but user or password is empty"); } } @@ -553,16 +565,12 @@ impl NatsCommon { .credentials(&self.create_credential(v_nkey, v_jwt)?) .expect("failed to parse static creds") } else { - return Err(anyhow_error!( - "nats connect mode is credential, but nkey or jwt is empty" - )); + bail!("nats connect mode is credential, but nkey or jwt is empty"); } } "plain" => {} _ => { - return Err(anyhow_error!( - "nats connect mode only accept user_and_password/credential/plain" - )); + bail!("nats connect mode only accept user_and_password/credential/plain"); } }; @@ -575,11 +583,12 @@ impl NatsCommon { .collect::, _>>()?, ) .await - .map_err(|e| SinkError::Nats(anyhow_error!("build nats client error: {:?}", e)))?; + .context("build nats client error") + .map_err(SinkError::Nats)?; Ok(client) } - pub(crate) async fn build_context(&self) -> anyhow::Result { + pub(crate) async fn build_context(&self) -> ConnectorResult { let client = self.build_client().await?; let jetstream = async_nats::jetstream::new(client); Ok(jetstream) @@ -590,7 +599,7 @@ impl NatsCommon { stream: String, split_id: String, start_sequence: NatsOffset, - ) -> anyhow::Result< + ) -> ConnectorResult< async_nats::jetstream::consumer::Consumer, > { let context = self.build_context().await?; @@ -609,13 +618,16 @@ impl NatsCommon { NatsOffset::Earliest => DeliverPolicy::All, NatsOffset::Latest => DeliverPolicy::Last, NatsOffset::SequenceNumber(v) => { - let parsed = v.parse::()?; + let parsed = v + .parse::() + .context("failed to parse nats offset as sequence number")?; DeliverPolicy::ByStartSequence { start_sequence: 1 + parsed, } } NatsOffset::Timestamp(v) => DeliverPolicy::ByStartTime { - start_time: OffsetDateTime::from_unix_timestamp_nanos(v * 1_000_000)?, + start_time: OffsetDateTime::from_unix_timestamp_nanos(v * 1_000_000) + .context("invalid timestamp for nats offset")?, }, NatsOffset::None => DeliverPolicy::All, }; @@ -632,7 +644,7 @@ impl NatsCommon { &self, jetstream: jetstream::Context, stream: String, - ) -> anyhow::Result { + ) -> ConnectorResult { let subjects: Vec = self.subject.split(',').map(|s| s.to_string()).collect(); let mut config = jetstream::stream::Config { name: stream, @@ -659,7 +671,7 @@ impl NatsCommon { Ok(stream) } - pub(crate) fn create_credential(&self, seed: &str, jwt: &str) -> anyhow::Result { + pub(crate) fn create_credential(&self, seed: &str, jwt: &str) -> ConnectorResult { let creds = format!( "-----BEGIN NATS USER JWT-----\n{}\n------END NATS USER JWT------\n\n\ ************************* IMPORTANT *************************\n\ diff --git a/src/connector/src/error.rs b/src/connector/src/error.rs index 86ba8a6cc39bc..3dc10af3d8e7a 100644 --- a/src/connector/src/error.rs +++ b/src/connector/src/error.rs @@ -12,57 +12,58 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::{ErrorCode, RwError}; -use thiserror::Error; +use risingwave_common::error::v2::def_anyhow_newtype; +use risingwave_pb::PbFieldNotFound; +use risingwave_rpc_client::error::RpcError; -#[derive(Error, Debug)] -pub enum ConnectorError { - #[error("Parse error: {0}")] - Parse(&'static str), +use crate::parser::AccessError; +use crate::schema::schema_registry::{ConcurrentRequestError, WireFormatError}; +use crate::schema::InvalidOptionError; +use crate::sink::SinkError; - #[error("Invalid parameter {name}: {reason}")] - InvalidParam { name: &'static str, reason: String }, +def_anyhow_newtype! { + pub ConnectorError, - #[error("Kafka error: {0}")] - Kafka(#[from] rdkafka::error::KafkaError), + // Common errors + std::io::Error => transparent, - #[error("Config error: {0}")] - Config( - #[source] - #[backtrace] - anyhow::Error, - ), + // Fine-grained connector errors + AccessError => transparent, + WireFormatError => transparent, + ConcurrentRequestError => transparent, + InvalidOptionError => transparent, + SinkError => transparent, + PbFieldNotFound => transparent, - #[error("Connection error: {0}")] - Connection( - #[source] - #[backtrace] - anyhow::Error, - ), + // TODO(error-handling): Remove implicit contexts below and specify ad-hoc context for each conversion. - #[error("MySQL error: {0}")] - MySql(#[from] mysql_async::Error), + // Parsing errors + url::ParseError => "failed to parse url", + serde_json::Error => "failed to parse json", + csv::Error => "failed to parse csv", - #[error("Postgres error: {0}")] - Postgres(#[from] tokio_postgres::Error), + // Connector errors + opendal::Error => transparent, // believed to be self-explanatory - #[error("Pulsar error: {0}")] - Pulsar( - #[source] - #[backtrace] - anyhow::Error, - ), - - #[error(transparent)] - Internal( - #[from] - #[backtrace] - anyhow::Error, - ), + mysql_async::Error => "MySQL error", + tokio_postgres::Error => "Postgres error", + apache_avro::Error => "Avro error", + rdkafka::error::KafkaError => "Kafka error", + pulsar::Error => "Pulsar error", + async_nats::jetstream::consumer::StreamError => "Nats error", + async_nats::jetstream::consumer::pull::MessagesError => "Nats error", + async_nats::jetstream::context::CreateStreamError => "Nats error", + async_nats::jetstream::stream::ConsumerError => "Nats error", + icelake::Error => "Iceberg error", + redis::RedisError => "Redis error", + arrow_schema::ArrowError => "Arrow error", + google_cloud_pubsub::client::google_cloud_auth::error::Error => "Google Cloud error", } -impl From for RwError { - fn from(s: ConnectorError) -> Self { - ErrorCode::ConnectorError(Box::new(s)).into() +pub type ConnectorResult = std::result::Result; + +impl From for RpcError { + fn from(value: ConnectorError) -> Self { + RpcError::Internal(value.0) } } diff --git a/src/connector/src/lib.rs b/src/connector/src/lib.rs index ae478b9c100ee..4a437755c5185 100644 --- a/src/connector/src/lib.rs +++ b/src/connector/src/lib.rs @@ -33,7 +33,7 @@ #![feature(error_generic_member_access)] #![feature(register_tool)] #![register_tool(rw)] -#![allow(rw::format_error)] // TODO(error-handling): need further refactoring +#![recursion_limit = "256"] use std::time::Duration; diff --git a/src/connector/src/macros.rs b/src/connector/src/macros.rs index 9a2383dbb4a96..4b375254c5ad1 100644 --- a/src/connector/src/macros.rs +++ b/src/connector/src/macros.rs @@ -20,7 +20,8 @@ macro_rules! for_all_classified_sources { { { Mysql }, { Postgres }, - { Citus } + { Citus }, + { Mongodb } }, // other sources // todo: file source do not nest with mq source. @@ -36,7 +37,8 @@ macro_rules! for_all_classified_sources { { Gcs, $crate::source::filesystem::opendal_source::GcsProperties , $crate::source::filesystem::OpendalFsSplit<$crate::source::filesystem::opendal_source::OpendalGcs> }, { OpendalS3, $crate::source::filesystem::opendal_source::OpendalS3Properties, $crate::source::filesystem::OpendalFsSplit<$crate::source::filesystem::opendal_source::OpendalS3> }, { PosixFs, $crate::source::filesystem::opendal_source::PosixFsProperties, $crate::source::filesystem::OpendalFsSplit<$crate::source::filesystem::opendal_source::OpendalPosixFs> }, - { Test, $crate::source::test_source::TestSourceProperties, $crate::source::test_source::TestSourceSplit} + { Test, $crate::source::test_source::TestSourceProperties, $crate::source::test_source::TestSourceSplit}, + { Iceberg, $crate::source::iceberg::IcebergProperties, $crate::source::iceberg::IcebergSplit} } $( ,$extra_args @@ -167,12 +169,12 @@ macro_rules! impl_split { $( impl TryFrom for $split { - type Error = anyhow::Error; + type Error = $crate::error::ConnectorError; fn try_from(split: SplitImpl) -> std::result::Result { match split { SplitImpl::$variant_name(inner) => Ok(inner), - other => Err(anyhow::anyhow!("expect {} but get {:?}", stringify!($split), other)) + other => risingwave_common::bail!("expect {} but get {:?}", stringify!($split), other), } } } diff --git a/src/connector/src/parser/additional_columns.rs b/src/connector/src/parser/additional_columns.rs index f604f1225227c..06cc061566690 100644 --- a/src/connector/src/parser/additional_columns.rs +++ b/src/connector/src/parser/additional_columns.rs @@ -12,104 +12,242 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::{HashMap, HashSet}; +use std::sync::LazyLock; + +use risingwave_common::bail; use risingwave_common::catalog::{ColumnCatalog, ColumnDesc, ColumnId}; use risingwave_common::types::{DataType, StructType}; -use risingwave_pb::plan_common::AdditionalColumnType; +use risingwave_pb::data::data_type::TypeName; +use risingwave_pb::data::DataType as PbDataType; +use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; +use risingwave_pb::plan_common::{ + AdditionalColumn, AdditionalColumnFilename, AdditionalColumnHeader, AdditionalColumnHeaders, + AdditionalColumnKey, AdditionalColumnOffset, AdditionalColumnPartition, + AdditionalColumnTimestamp, +}; +use crate::error::ConnectorResult; use crate::source::{ GCS_CONNECTOR, KAFKA_CONNECTOR, KINESIS_CONNECTOR, OPENDAL_S3_CONNECTOR, PULSAR_CONNECTOR, S3_CONNECTOR, }; -pub type CompatibleAdditionalColumnsFn = - Box ColumnCatalog + Send + Sync + 'static>; +// Hidden additional columns connectors which do not support `include` syntax. +pub static COMMON_COMPATIBLE_ADDITIONAL_COLUMNS: LazyLock> = + LazyLock::new(|| HashSet::from(["partition", "offset"])); + +pub static COMPATIBLE_ADDITIONAL_COLUMNS: LazyLock>> = + LazyLock::new(|| { + HashMap::from([ + ( + KAFKA_CONNECTOR, + HashSet::from(["key", "timestamp", "partition", "offset", "header"]), + ), + ( + PULSAR_CONNECTOR, + HashSet::from(["key", "partition", "offset"]), + ), + ( + KINESIS_CONNECTOR, + HashSet::from(["key", "partition", "offset", "timestamp"]), + ), + (OPENDAL_S3_CONNECTOR, HashSet::from(["file", "offset"])), + (S3_CONNECTOR, HashSet::from(["file", "offset"])), + (GCS_CONNECTOR, HashSet::from(["file", "offset"])), + ]) + }); + +pub fn gen_default_addition_col_name( + connector_name: &str, + additional_col_type: &str, + inner_field_name: Option<&str>, + data_type: Option<&str>, +) -> String { + let col_name = [ + Some(connector_name), + Some(additional_col_type), + inner_field_name, + data_type, + ]; + col_name.iter().fold("_rw".to_string(), |name, ele| { + if let Some(ele) = ele { + format!("{}_{}", name, ele) + } else { + name + } + }) +} -pub fn get_connector_compatible_additional_columns( +pub fn build_additional_column_catalog( + column_id: ColumnId, connector_name: &str, -) -> Option> { - let compatible_columns = match connector_name { - KAFKA_CONNECTOR => kafka_compatible_column_vec(), - PULSAR_CONNECTOR => pulsar_compatible_column_vec(), - KINESIS_CONNECTOR => kinesis_compatible_column_vec(), - OPENDAL_S3_CONNECTOR | S3_CONNECTOR | GCS_CONNECTOR => s3_compatible_column_column_vec(), - _ => return None, + additional_col_type: &str, + column_alias: Option, + inner_field_name: Option<&str>, + data_type: Option<&str>, + reject_unknown_connector: bool, +) -> ConnectorResult { + let compatible_columns = match ( + COMPATIBLE_ADDITIONAL_COLUMNS.get(connector_name), + reject_unknown_connector, + ) { + (Some(compat_cols), _) => compat_cols, + (None, false) => &COMMON_COMPATIBLE_ADDITIONAL_COLUMNS, + (None, true) => { + bail!( + "additional column is not supported for connector {}, acceptable connectors: {:?}", + connector_name, + COMPATIBLE_ADDITIONAL_COLUMNS.keys(), + ); + } }; - Some(compatible_columns) + if !compatible_columns.contains(additional_col_type) { + bail!( + "additional column type {} is not supported for connector {}, acceptable column types: {:?}", + additional_col_type, connector_name, compatible_columns + ); + } + + let column_name = column_alias.unwrap_or_else(|| { + gen_default_addition_col_name( + connector_name, + additional_col_type, + inner_field_name, + data_type, + ) + }); + + let catalog = match additional_col_type { + "key" => ColumnCatalog { + column_desc: ColumnDesc::named_with_additional_column( + column_name, + column_id, + DataType::Bytea, + AdditionalColumn { + column_type: Some(AdditionalColumnType::Key(AdditionalColumnKey {})), + }, + ), + is_hidden: false, + }, + "timestamp" => ColumnCatalog { + column_desc: ColumnDesc::named_with_additional_column( + column_name, + column_id, + DataType::Timestamptz, + AdditionalColumn { + column_type: Some(AdditionalColumnType::Timestamp( + AdditionalColumnTimestamp {}, + )), + }, + ), + is_hidden: false, + }, + "partition" => ColumnCatalog { + column_desc: ColumnDesc::named_with_additional_column( + column_name, + column_id, + DataType::Varchar, + AdditionalColumn { + column_type: Some(AdditionalColumnType::Partition( + AdditionalColumnPartition {}, + )), + }, + ), + is_hidden: false, + }, + "offset" => ColumnCatalog { + column_desc: ColumnDesc::named_with_additional_column( + column_name, + column_id, + DataType::Varchar, + AdditionalColumn { + column_type: Some(AdditionalColumnType::Offset(AdditionalColumnOffset {})), + }, + ), + is_hidden: false, + }, + "file" => ColumnCatalog { + column_desc: ColumnDesc::named_with_additional_column( + column_name, + column_id, + DataType::Varchar, + AdditionalColumn { + column_type: Some(AdditionalColumnType::Filename(AdditionalColumnFilename {})), + }, + ), + is_hidden: false, + }, + "header" => build_header_catalog(column_id, &column_name, inner_field_name, data_type), + _ => unreachable!(), + }; + + Ok(catalog) } -fn kafka_compatible_column_vec() -> Vec<(&'static str, CompatibleAdditionalColumnsFn)> { - vec![ - ( - "key", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, +fn build_header_catalog( + column_id: ColumnId, + col_name: &str, + inner_field_name: Option<&str>, + data_type: Option<&str>, +) -> ColumnCatalog { + if let Some(inner) = inner_field_name { + let (data_type, pb_data_type) = { + if let Some(type_name) = data_type { + match type_name { + "bytea" => ( DataType::Bytea, - AdditionalColumnType::Key, - ), - is_hidden: false, - } - }), - ), - ( - "timestamp", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Timestamptz, - AdditionalColumnType::Timestamp, + PbDataType { + type_name: TypeName::Bytea as i32, + ..Default::default() + }, ), - is_hidden: false, - } - }), - ), - ( - "partition", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Varchar, - AdditionalColumnType::Partition, - ), - is_hidden: false, - } - }), - ), - ( - "offset", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, + "varchar" => ( DataType::Varchar, - AdditionalColumnType::Offset, - ), - is_hidden: false, - } - }), - ), - ( - "header", // type: struct[] - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::List(get_kafka_header_item_datatype().into()), - AdditionalColumnType::Header, + PbDataType { + type_name: TypeName::Varchar as i32, + ..Default::default() + }, ), - is_hidden: false, + _ => unreachable!(), } - }), - ), - ] + } else { + ( + DataType::Bytea, + PbDataType { + type_name: TypeName::Bytea as i32, + ..Default::default() + }, + ) + } + }; + ColumnCatalog { + column_desc: ColumnDesc::named_with_additional_column( + col_name, + column_id, + data_type, + AdditionalColumn { + column_type: Some(AdditionalColumnType::HeaderInner(AdditionalColumnHeader { + inner_field: inner.to_string(), + data_type: Some(pb_data_type), + })), + }, + ), + is_hidden: false, + } + } else { + ColumnCatalog { + column_desc: ColumnDesc::named_with_additional_column( + col_name, + column_id, + DataType::List(get_kafka_header_item_datatype().into()), + AdditionalColumn { + column_type: Some(AdditionalColumnType::Headers(AdditionalColumnHeaders {})), + }, + ), + is_hidden: false, + } + } } pub fn get_kafka_header_item_datatype() -> DataType { @@ -117,143 +255,23 @@ pub fn get_kafka_header_item_datatype() -> DataType { DataType::Struct(StructType::new(struct_inner)) } -fn pulsar_compatible_column_vec() -> Vec<(&'static str, CompatibleAdditionalColumnsFn)> { - vec![ - ( - "key", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Bytea, - AdditionalColumnType::Key, - ), - is_hidden: false, - } - }), - ), - ( - "partition", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Varchar, - AdditionalColumnType::Partition, - ), - is_hidden: false, - } - }), - ), - ( - "offset", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Varchar, - AdditionalColumnType::Offset, - ), - is_hidden: false, - } - }), - ), - ] -} +#[cfg(test)] +mod test { + use super::*; -fn kinesis_compatible_column_vec() -> Vec<(&'static str, CompatibleAdditionalColumnsFn)> { - vec![ - ( - "key", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Bytea, - AdditionalColumnType::Key, - ), - is_hidden: false, - } - }), - ), - ( - "partition", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Varchar, - AdditionalColumnType::Partition, - ), - is_hidden: false, - } - }), - ), - ( - "offset", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Varchar, - AdditionalColumnType::Offset, - ), - is_hidden: false, - } - }), - ), - ( - "timestamp", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Timestamptz, - AdditionalColumnType::Timestamp, - ), - is_hidden: false, - } - }), - ), - ] -} - -fn s3_compatible_column_column_vec() -> Vec<(&'static str, CompatibleAdditionalColumnsFn)> { - vec![ - ( - "file", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Varchar, - AdditionalColumnType::Filename, - ), - is_hidden: false, - } - }), - ), - ( - "offset", - Box::new(|id: ColumnId, name: &str| -> ColumnCatalog { - ColumnCatalog { - column_desc: ColumnDesc::named_with_additional_column( - name, - id, - DataType::Varchar, - AdditionalColumnType::Offset, - ), - is_hidden: false, - } - }), - ), - ] + #[test] + fn test_gen_default_addition_col_name() { + assert_eq!( + gen_default_addition_col_name("kafka", "key", None, None), + "_rw_kafka_key" + ); + assert_eq!( + gen_default_addition_col_name("kafka", "header", Some("inner"), None), + "_rw_kafka_header_inner" + ); + assert_eq!( + gen_default_addition_col_name("kafka", "header", Some("inner"), Some("varchar")), + "_rw_kafka_header_inner_varchar" + ); + } } diff --git a/src/connector/src/parser/avro/parser.rs b/src/connector/src/parser/avro/parser.rs index 10e000a4fdab7..7343f1c43118c 100644 --- a/src/connector/src/parser/avro/parser.rs +++ b/src/connector/src/parser/avro/parser.rs @@ -15,15 +15,15 @@ use std::fmt::Debug; use std::sync::Arc; +use anyhow::Context; use apache_avro::types::Value; use apache_avro::{from_avro_datum, Reader, Schema}; -use risingwave_common::error::ErrorCode::{InternalError, ProtocolError}; -use risingwave_common::error::{Result, RwError}; -use risingwave_common::try_match_expand; +use risingwave_common::{bail, try_match_expand}; use risingwave_pb::plan_common::ColumnDesc; use super::schema_resolver::ConfluentSchemaResolver; use super::util::avro_schema_to_column_descs; +use crate::error::ConnectorResult; use crate::parser::unified::avro::{AvroAccess, AvroParseOptions}; use crate::parser::unified::AccessImpl; use crate::parser::util::bytes_from_url; @@ -41,7 +41,7 @@ pub struct AvroAccessBuilder { } impl AccessBuilder for AvroAccessBuilder { - async fn generate_accessor(&mut self, payload: Vec) -> Result> { + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult> { self.value = self.parse_avro_value(&payload, Some(&*self.schema)).await?; Ok(AccessImpl::Avro(AvroAccess::new( self.value.as_ref().unwrap(), @@ -51,7 +51,7 @@ impl AccessBuilder for AvroAccessBuilder { } impl AvroAccessBuilder { - pub fn new(config: AvroParserConfig, encoding_type: EncodingType) -> Result { + pub fn new(config: AvroParserConfig, encoding_type: EncodingType) -> ConnectorResult { let AvroParserConfig { schema, key_schema, @@ -60,9 +60,7 @@ impl AvroAccessBuilder { } = config; Ok(Self { schema: match encoding_type { - EncodingType::Key => key_schema.ok_or(RwError::from(ProtocolError( - "Avro with empty key schema".to_string(), - )))?, + EncodingType::Key => key_schema.context("Avro with empty key schema")?, EncodingType::Value => schema, }, schema_resolver, @@ -74,7 +72,7 @@ impl AvroAccessBuilder { &self, payload: &[u8], reader_schema: Option<&Schema>, - ) -> anyhow::Result> { + ) -> ConnectorResult> { // parse payload to avro value // if use confluent schema, get writer schema from confluent schema registry if let Some(resolver) = &self.schema_resolver { @@ -90,9 +88,7 @@ impl AvroAccessBuilder { match reader.next() { Some(Ok(v)) => Ok(Some(v)), Some(Err(e)) => Err(e)?, - None => { - anyhow::bail!("avro parse unexpected eof") - } + None => bail!("avro parse unexpected eof"), } } else { unreachable!("both schema_resolver and reader_schema not exist"); @@ -108,7 +104,7 @@ pub struct AvroParserConfig { } impl AvroParserConfig { - pub async fn new(encoding_properties: EncodingProperties) -> Result { + pub async fn new(encoding_properties: EncodingProperties) -> ConnectorResult { let avro_config = try_match_expand!(encoding_properties, EncodingProperties::Avro)?; let schema_location = &avro_config.row_schema_location; let enable_upsert = avro_config.enable_upsert; @@ -126,9 +122,7 @@ impl AvroParserConfig { )?) } else { if let Some(name) = &avro_config.key_record_name { - return Err(RwError::from(ProtocolError(format!( - "key.message = {name} not used", - )))); + bail!("key.message = {name} not used"); } None }; @@ -151,15 +145,12 @@ impl AvroParserConfig { }) } else { if enable_upsert { - return Err(RwError::from(InternalError( - "avro upsert without schema registry is not supported".to_string(), - ))); + bail!("avro upsert without schema registry is not supported"); } let url = url.first().unwrap(); let schema_content = bytes_from_url(url, avro_config.aws_auth_props.as_ref()).await?; - let schema = Schema::parse_reader(&mut schema_content.as_slice()).map_err(|e| { - RwError::from(InternalError(format!("Avro schema parse error {}", e))) - })?; + let schema = Schema::parse_reader(&mut schema_content.as_slice()) + .context("failed to parse avro schema")?; Ok(Self { schema: Arc::new(schema), key_schema: None, @@ -168,7 +159,7 @@ impl AvroParserConfig { } } - pub fn extract_pks(&self) -> anyhow::Result> { + pub fn extract_pks(&self) -> ConnectorResult> { avro_schema_to_column_descs( self.key_schema .as_deref() @@ -176,7 +167,7 @@ impl AvroParserConfig { ) } - pub fn map_to_columns(&self) -> anyhow::Result> { + pub fn map_to_columns(&self) -> ConnectorResult> { avro_schema_to_column_descs(self.schema.as_ref()) } } @@ -204,6 +195,7 @@ mod test { use super::*; use crate::common::AwsAuthProps; + use crate::error::ConnectorResult; use crate::parser::plain_parser::PlainParser; use crate::parser::unified::avro::unix_epoch_days; use crate::parser::{ @@ -264,7 +256,7 @@ mod test { println!("schema = {:?}", schema.unwrap()); } - async fn new_avro_conf_from_local(file_name: &str) -> Result { + async fn new_avro_conf_from_local(file_name: &str) -> ConnectorResult { let schema_path = "file://".to_owned() + &test_data_path(file_name); let info = StreamSourceInfo { row_schema_location: schema_path.clone(), @@ -277,7 +269,7 @@ mod test { AvroParserConfig::new(parser_config.encoding_config).await } - async fn new_avro_parser_from_local(file_name: &str) -> Result { + async fn new_avro_parser_from_local(file_name: &str) -> ConnectorResult { let conf = new_avro_conf_from_local(file_name).await?; Ok(PlainParser { diff --git a/src/connector/src/parser/avro/schema_resolver.rs b/src/connector/src/parser/avro/schema_resolver.rs index ffd821ec37a4c..cdc52de7accee 100644 --- a/src/connector/src/parser/avro/schema_resolver.rs +++ b/src/connector/src/parser/avro/schema_resolver.rs @@ -14,11 +14,11 @@ use std::sync::Arc; +use anyhow::Context; use apache_avro::Schema; use moka::future::Cache; -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; +use crate::error::ConnectorResult; use crate::schema::schema_registry::{Client, ConfluentSchema}; #[derive(Debug)] @@ -28,9 +28,12 @@ pub struct ConfluentSchemaResolver { } impl ConfluentSchemaResolver { - async fn parse_and_cache_schema(&self, raw_schema: ConfluentSchema) -> Result> { - let schema = Schema::parse_str(&raw_schema.content) - .map_err(|e| RwError::from(ProtocolError(format!("Avro schema parse error {}", e))))?; + async fn parse_and_cache_schema( + &self, + raw_schema: ConfluentSchema, + ) -> ConnectorResult> { + let schema = + Schema::parse_str(&raw_schema.content).context("failed to parse avro schema")?; let schema = Arc::new(schema); self.writer_schemas .insert(raw_schema.id, Arc::clone(&schema)) @@ -46,7 +49,7 @@ impl ConfluentSchemaResolver { } } - pub async fn get_by_subject_name(&self, subject_name: &str) -> Result> { + pub async fn get_by_subject_name(&self, subject_name: &str) -> ConnectorResult> { let raw_schema = self.get_raw_schema_by_subject_name(subject_name).await?; self.parse_and_cache_schema(raw_schema).await } @@ -54,14 +57,15 @@ impl ConfluentSchemaResolver { pub async fn get_raw_schema_by_subject_name( &self, subject_name: &str, - ) -> Result { + ) -> ConnectorResult { self.confluent_client .get_schema_by_subject(subject_name) .await + .map_err(Into::into) } // get the writer schema by id - pub async fn get(&self, schema_id: i32) -> Result> { + pub async fn get(&self, schema_id: i32) -> ConnectorResult> { // TODO: use `get_with` if let Some(schema) = self.writer_schemas.get(&schema_id).await { Ok(schema) diff --git a/src/connector/src/parser/avro/util.rs b/src/connector/src/parser/avro/util.rs index f81f73fd1b7e4..a58ad884fd886 100644 --- a/src/connector/src/parser/avro/util.rs +++ b/src/connector/src/parser/avro/util.rs @@ -16,20 +16,23 @@ use std::sync::LazyLock; use apache_avro::schema::{DecimalSchema, RecordSchema, Schema}; use itertools::Itertools; +use risingwave_common::bail; use risingwave_common::log::LogSuppresser; use risingwave_common::types::{DataType, Decimal}; -use risingwave_pb::plan_common::{AdditionalColumnType, ColumnDesc, ColumnDescVersion}; +use risingwave_pb::plan_common::{AdditionalColumn, ColumnDesc, ColumnDescVersion}; -pub fn avro_schema_to_column_descs(schema: &Schema) -> anyhow::Result> { +use crate::error::ConnectorResult; + +pub fn avro_schema_to_column_descs(schema: &Schema) -> ConnectorResult> { if let Schema::Record(RecordSchema { fields, .. }) = schema { let mut index = 0; let fields = fields .iter() .map(|field| avro_field_to_column_desc(&field.name, &field.schema, &mut index)) - .collect::>>()?; + .collect::>>()?; Ok(fields) } else { - anyhow::bail!("schema invalid, record type required at top level of the schema."); + bail!("schema invalid, record type required at top level of the schema."); } } @@ -40,7 +43,7 @@ fn avro_field_to_column_desc( name: &str, schema: &Schema, index: &mut i32, -) -> anyhow::Result { +) -> ConnectorResult { let data_type = avro_type_mapping(schema)?; match schema { Schema::Record(RecordSchema { @@ -51,7 +54,7 @@ fn avro_field_to_column_desc( let vec_column = fields .iter() .map(|f| avro_field_to_column_desc(&f.name, &f.schema, index)) - .collect::>>()?; + .collect::>>()?; *index += 1; Ok(ColumnDesc { column_type: Some(data_type.to_protobuf()), @@ -61,7 +64,8 @@ fn avro_field_to_column_desc( type_name: schema_name.to_string(), generated_or_default_column: None, description: None, - additional_column_type: AdditionalColumnType::Normal as i32, + additional_column_type: 0, // deprecated + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, }) } @@ -71,7 +75,7 @@ fn avro_field_to_column_desc( column_type: Some(data_type.to_protobuf()), column_id: *index, name: name.to_owned(), - additional_column_type: AdditionalColumnType::Normal as i32, + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, ..Default::default() }) @@ -79,7 +83,7 @@ fn avro_field_to_column_desc( } } -fn avro_type_mapping(schema: &Schema) -> anyhow::Result { +fn avro_type_mapping(schema: &Schema) -> ConnectorResult { let data_type = match schema { Schema::String => DataType::Varchar, Schema::Int => DataType::Int32, @@ -93,16 +97,18 @@ fn avro_type_mapping(schema: &Schema) -> anyhow::Result { LazyLock::new(LogSuppresser::default); if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::warn!( - "RisingWave supports decimal precision up to {}, but got {}. Will truncate. ({} suppressed)", - Decimal::MAX_PRECISION, - suppressed_count, - precision - ); + suppressed_count, + "RisingWave supports decimal precision up to {}, but got {}. Will truncate.", + Decimal::MAX_PRECISION, + precision + ); } } DataType::Decimal } Schema::Date => DataType::Date, + Schema::LocalTimestampMillis => DataType::Timestamp, + Schema::LocalTimestampMicros => DataType::Timestamp, Schema::TimestampMillis => DataType::Timestamptz, Schema::TimestampMicros => DataType::Timestamptz, Schema::Duration => DataType::Interval, @@ -120,7 +126,7 @@ fn avro_type_mapping(schema: &Schema) -> anyhow::Result { let struct_fields = fields .iter() .map(|f| avro_type_mapping(&f.schema)) - .collect::>>()?; + .collect::>>()?; let struct_names = fields.iter().map(|f| f.name.clone()).collect_vec(); DataType::new_struct(struct_fields, struct_names) } @@ -139,12 +145,16 @@ fn avro_type_mapping(schema: &Schema) -> anyhow::Result { avro_type_mapping(nested_schema)? } - _ => { - return Err(anyhow::format_err!( - "unsupported type in Avro: {:?}", - schema - )); + Schema::Ref { name } => { + if name.name == DBZ_VARIABLE_SCALE_DECIMAL_NAME + && name.namespace == Some(DBZ_VARIABLE_SCALE_DECIMAL_NAMESPACE.into()) + { + DataType::Decimal + } else { + bail!("unsupported type in Avro: {:?}", schema); + } } + _ => bail!("unsupported type in Avro: {:?}", schema), }; Ok(data_type) diff --git a/src/connector/src/parser/bytes_parser.rs b/src/connector/src/parser/bytes_parser.rs index 07420a2d36ab1..4f353ce2c60e6 100644 --- a/src/connector/src/parser/bytes_parser.rs +++ b/src/connector/src/parser/bytes_parser.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::Result; use risingwave_common::try_match_expand; use super::unified::bytes::BytesAccess; use super::unified::AccessImpl; use super::{AccessBuilder, EncodingProperties}; +use crate::error::ConnectorResult; #[derive(Debug)] pub struct BytesAccessBuilder { @@ -26,7 +26,7 @@ pub struct BytesAccessBuilder { impl AccessBuilder for BytesAccessBuilder { #[allow(clippy::unused_async)] - async fn generate_accessor(&mut self, payload: Vec) -> Result> { + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult> { Ok(AccessImpl::Bytes(BytesAccess::new( &self.column_name, payload, @@ -35,7 +35,7 @@ impl AccessBuilder for BytesAccessBuilder { } impl BytesAccessBuilder { - pub fn new(encoding_properties: EncodingProperties) -> Result { + pub fn new(encoding_properties: EncodingProperties) -> ConnectorResult { let config = try_match_expand!(encoding_properties, EncodingProperties::Bytes)?; Ok(Self { column_name: config.column_name, diff --git a/src/connector/src/parser/canal/simd_json_parser.rs b/src/connector/src/parser/canal/simd_json_parser.rs index fdb317c96c20a..75e6656fd7a7a 100644 --- a/src/connector/src/parser/canal/simd_json_parser.rs +++ b/src/connector/src/parser/canal/simd_json_parser.rs @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use anyhow::Context; use itertools::Itertools; -use risingwave_common::error::ErrorCode::{self, ProtocolError}; -use risingwave_common::error::{Result, RwError}; +use risingwave_common::bail; use simd_json::prelude::{MutableObject, ValueAsScalar, ValueObjectAccess}; use simd_json::BorrowedValue; +use crate::error::ConnectorResult; use crate::only_parse_payload; use crate::parser::canal::operators::*; use crate::parser::unified::json::{JsonAccess, JsonParseOptions}; @@ -44,7 +45,7 @@ impl CanalJsonParser { rw_columns: Vec, source_ctx: SourceContextRef, config: &JsonProperties, - ) -> Result { + ) -> ConnectorResult { Ok(Self { rw_columns, source_ctx, @@ -57,29 +58,23 @@ impl CanalJsonParser { &self, mut payload: Vec, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result<()> { + ) -> ConnectorResult<()> { let mut event: BorrowedValue<'_> = simd_json::to_borrowed_value(&mut payload[self.payload_start_idx..]) - .map_err(|e| RwError::from(ProtocolError(e.to_string())))?; + .context("failed to parse canal json payload")?; - let is_ddl = event.get(IS_DDL).and_then(|v| v.as_bool()).ok_or_else(|| { - RwError::from(ProtocolError( - "isDdl field not found in canal json".to_owned(), - )) - })?; + let is_ddl = event + .get(IS_DDL) + .and_then(|v| v.as_bool()) + .context("field `isDdl` not found in canal json")?; if is_ddl { - return Err(RwError::from(ProtocolError( - "received a DDL message, please set `canal.instance.filter.query.dml` to true." - .to_string(), - ))); + bail!("received a DDL message, please set `canal.instance.filter.query.dml` to true."); } let op = match event.get(OP).and_then(|v| v.as_str()) { Some(CANAL_INSERT_EVENT | CANAL_UPDATE_EVENT) => ChangeEventOperation::Upsert, Some(CANAL_DELETE_EVENT) => ChangeEventOperation::Delete, - _ => Err(RwError::from(ProtocolError( - "op field not found in canal json".to_owned(), - )))?, + _ => bail!("op field not found in canal json"), }; let events = event @@ -88,11 +83,7 @@ impl CanalJsonParser { BorrowedValue::Array(array) => Some(array), _ => None, }) - .ok_or_else(|| { - RwError::from(ProtocolError( - "'data' is missing for creating event".to_string(), - )) - })?; + .context("field `data` is missing for creating event")?; let mut errors = Vec::new(); for event in events.drain(..) { @@ -106,11 +97,12 @@ impl CanalJsonParser { if errors.is_empty() { Ok(()) } else { - Err(RwError::from(ErrorCode::InternalError(format!( + // TODO(error-handling): multiple errors + bail!( "failed to parse {} row(s) in a single canal json message: {}", errors.len(), - errors.iter().join(", ") - )))) + errors.iter().format(", ") + ); } } } @@ -133,7 +125,7 @@ impl ByteStreamSourceParser for CanalJsonParser { _key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { + ) -> ConnectorResult<()> { only_parse_payload!(self, payload, writer) } } diff --git a/src/connector/src/parser/csv_parser.rs b/src/connector/src/parser/csv_parser.rs index 77f0cdd30e41f..8a8bb211da327 100644 --- a/src/connector/src/parser/csv_parser.rs +++ b/src/connector/src/parser/csv_parser.rs @@ -12,12 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{ErrorCode, Result, RwError}; use risingwave_common::types::{Date, Decimal, Time, Timestamp, Timestamptz}; use super::unified::{AccessError, AccessResult}; use super::{ByteStreamSourceParser, CsvProperties}; +use crate::error::ConnectorResult; use crate::only_parse_payload; use crate::parser::{ParserFormat, SourceStreamChunkRowWriter}; use crate::source::{DataType, SourceColumnDesc, SourceContext, SourceContextRef}; @@ -46,7 +45,7 @@ impl CsvParser { rw_columns: Vec, csv_props: CsvProperties, source_ctx: SourceContextRef, - ) -> Result { + ) -> ConnectorResult { let CsvProperties { delimiter, has_header, @@ -60,15 +59,14 @@ impl CsvParser { }) } - fn read_row(&self, buf: &[u8]) -> Result> { + fn read_row(&self, buf: &[u8]) -> ConnectorResult> { let mut reader_builder = csv::ReaderBuilder::default(); reader_builder.delimiter(self.delimiter).has_headers(false); let record = reader_builder .from_reader(buf) .records() .next() - .transpose() - .map_err(|err| RwError::from(ProtocolError(err.to_string())))?; + .transpose()?; Ok(record .map(|record| record.iter().map(|field| field.to_string()).collect()) .unwrap_or_default()) @@ -105,7 +103,7 @@ impl CsvParser { &mut self, payload: Vec, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result<()> { + ) -> ConnectorResult<()> { let mut fields = self.read_row(&payload)?; if let Some(headers) = &mut self.headers { @@ -161,7 +159,7 @@ impl ByteStreamSourceParser for CsvParser { _key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { + ) -> ConnectorResult<()> { only_parse_payload!(self, payload, writer) } } diff --git a/src/connector/src/parser/debezium/avro_parser.rs b/src/connector/src/parser/debezium/avro_parser.rs index b4d1f0d145a75..ca1574af3d6b2 100644 --- a/src/connector/src/parser/debezium/avro_parser.rs +++ b/src/connector/src/parser/debezium/avro_parser.rs @@ -17,12 +17,11 @@ use std::sync::Arc; use apache_avro::types::Value; use apache_avro::{from_avro_datum, Schema}; -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; use risingwave_common::try_match_expand; use risingwave_pb::catalog::PbSchemaRegistryNameStrategy; use risingwave_pb::plan_common::ColumnDesc; +use crate::error::ConnectorResult; use crate::parser::avro::schema_resolver::ConfluentSchemaResolver; use crate::parser::avro::util::avro_schema_to_column_descs; use crate::parser::unified::avro::{ @@ -50,13 +49,10 @@ pub struct DebeziumAvroAccessBuilder { // TODO: reduce encodingtype match impl AccessBuilder for DebeziumAvroAccessBuilder { - async fn generate_accessor(&mut self, payload: Vec) -> Result> { + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult> { let (schema_id, mut raw_payload) = extract_schema_id(&payload)?; let schema = self.schema_resolver.get(schema_id).await?; - self.value = Some( - from_avro_datum(schema.as_ref(), &mut raw_payload, None) - .map_err(|e| RwError::from(ProtocolError(e.to_string())))?, - ); + self.value = Some(from_avro_datum(schema.as_ref(), &mut raw_payload, None)?); self.key_schema = match self.encoding_type { EncodingType::Key => Some(schema), EncodingType::Value => None, @@ -72,19 +68,19 @@ impl AccessBuilder for DebeziumAvroAccessBuilder { } impl DebeziumAvroAccessBuilder { - pub fn new(config: DebeziumAvroParserConfig, encoding_type: EncodingType) -> Result { + pub fn new( + config: DebeziumAvroParserConfig, + encoding_type: EncodingType, + ) -> ConnectorResult { let DebeziumAvroParserConfig { outer_schema, schema_resolver, .. } = config; - let resolver = apache_avro::schema::ResolvedSchema::try_from(&*outer_schema) - .map_err(|e| RwError::from(ProtocolError(e.to_string())))?; + let resolver = apache_avro::schema::ResolvedSchema::try_from(&*outer_schema)?; // todo: to_resolved may cause stackoverflow if there's a loop in the schema - let schema = resolver - .to_resolved(&outer_schema) - .map_err(|e| RwError::from(ProtocolError(e.to_string())))?; + let schema = resolver.to_resolved(&outer_schema)?; Ok(Self { schema, schema_resolver, @@ -104,7 +100,7 @@ pub struct DebeziumAvroParserConfig { } impl DebeziumAvroParserConfig { - pub async fn new(encoding_config: EncodingProperties) -> Result { + pub async fn new(encoding_config: EncodingProperties) -> ConnectorResult { let avro_config = try_match_expand!(encoding_config, EncodingProperties::Avro)?; let schema_location = &avro_config.row_schema_location; let client_config = &avro_config.client_config; @@ -126,11 +122,11 @@ impl DebeziumAvroParserConfig { }) } - pub fn extract_pks(&self) -> anyhow::Result> { + pub fn extract_pks(&self) -> ConnectorResult> { avro_schema_to_column_descs(&self.key_schema) } - pub fn map_to_columns(&self) -> anyhow::Result> { + pub fn map_to_columns(&self) -> ConnectorResult> { avro_schema_to_column_descs(avro_schema_skip_union(avro_extract_field_schema( &self.outer_schema, Some("before"), @@ -151,6 +147,7 @@ mod tests { use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_pb::catalog::StreamSourceInfo; + use risingwave_pb::data::data_type::TypeName; use risingwave_pb::plan_common::{PbEncodeType, PbFormatType}; use super::*; @@ -256,6 +253,64 @@ mod tests { assert_eq!(names, vec!["id".to_owned()]) } + #[test] + fn test_ref_avro_type() { + let test_schema_str = r#"{ + "type": "record", + "name": "Key", + "namespace": "dbserver1.inventory.customers", + "fields": [{ + "name": "id", + "type": "int" + }, + { + "name": "unconstrained_decimal", + "type": [ + "null", + { + "type": "record", + "name": "VariableScaleDecimal", + "namespace": "io.debezium.data", + "fields": [ + { + "name": "scale", + "type": "int" + }, + { + "name": "value", + "type": "bytes" + } + ], + "connect.doc": "Variable scaled decimal", + "connect.name": "io.debezium.data.VariableScaleDecimal", + "connect.version": 1 + } + ], + "default": null + }, + { + "name": "unconstrained_numeric", + "type": [ + "null", + "io.debezium.data.VariableScaleDecimal" + ], + "default": null + } + ], + "connect.name": "dbserver1.inventory.customers.Key" +} +"#; + let schema = Schema::parse_str(test_schema_str).unwrap(); + let columns = avro_schema_to_column_descs(&schema).unwrap(); + for col in &columns { + let dtype = col.column_type.as_ref().unwrap(); + println!("name = {}, type = {:?}", col.name, dtype.type_name); + if col.name.contains("unconstrained") { + assert_eq!(dtype.type_name, TypeName::Decimal as i32); + } + } + } + #[test] fn test_map_to_columns() { let outer_schema = get_outer_schema(); @@ -294,7 +349,7 @@ mod tests { #[ignore] #[tokio::test] - async fn test_debezium_avro_parser() -> Result<()> { + async fn test_debezium_avro_parser() -> crate::error::ConnectorResult<()> { let props = convert_args!(hashmap!( "kafka.topic" => "dbserver1.inventory.customers" )); diff --git a/src/connector/src/parser/debezium/debezium_parser.rs b/src/connector/src/parser/debezium/debezium_parser.rs index 0f79677860f8d..40f3be7e233b3 100644 --- a/src/connector/src/parser/debezium/debezium_parser.rs +++ b/src/connector/src/parser/debezium/debezium_parser.rs @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; +use std::collections::BTreeMap; + +use risingwave_common::bail; use super::simd_json_parser::DebeziumJsonAccessBuilder; use super::{DebeziumAvroAccessBuilder, DebeziumAvroParserConfig}; +use crate::error::ConnectorResult; use crate::extract_key_config; use crate::parser::unified::debezium::DebeziumChangeEvent; use crate::parser::unified::util::apply_row_operation_on_stream_chunk_writer; @@ -33,12 +35,33 @@ pub struct DebeziumParser { payload_builder: AccessBuilderImpl, pub(crate) rw_columns: Vec, source_ctx: SourceContextRef, + + props: DebeziumProps, +} + +pub const DEBEZIUM_IGNORE_KEY: &str = "ignore_key"; + +#[derive(Debug, Clone, Default)] +pub struct DebeziumProps { + // Ignore the key part of the message. + // If enabled, we don't take the key part into message accessor. + pub ignore_key: bool, +} + +impl DebeziumProps { + pub fn from(props: &BTreeMap) -> Self { + let ignore_key = props + .get(DEBEZIUM_IGNORE_KEY) + .map(|v| v.eq_ignore_ascii_case("true")) + .unwrap_or(false); + Self { ignore_key } + } } async fn build_accessor_builder( config: EncodingProperties, encoding_type: EncodingType, -) -> Result { +) -> ConnectorResult { match config { EncodingProperties::Avro(_) => { let config = DebeziumAvroParserConfig::new(config).await?; @@ -52,9 +75,7 @@ async fn build_accessor_builder( EncodingProperties::Protobuf(_) => { Ok(AccessBuilderImpl::new_default(config, encoding_type).await?) } - _ => Err(RwError::from(ProtocolError( - "unsupported encoding for Debezium".to_string(), - ))), + _ => bail!("unsupported encoding for Debezium"), } } @@ -63,26 +84,35 @@ impl DebeziumParser { props: SpecificParserConfig, rw_columns: Vec, source_ctx: SourceContextRef, - ) -> Result { + ) -> ConnectorResult { let (key_config, key_type) = extract_key_config!(props); let key_builder = build_accessor_builder(key_config, key_type).await?; let payload_builder = build_accessor_builder(props.encoding_config, EncodingType::Value).await?; + let debezium_props = if let ProtocolProperties::Debezium(props) = props.protocol_config { + props + } else { + unreachable!( + "expecting Debezium protocol properties but got {:?}", + props.protocol_config + ) + }; Ok(Self { key_builder, payload_builder, rw_columns, source_ctx, + props: debezium_props, }) } - pub async fn new_for_test(rw_columns: Vec) -> Result { + pub async fn new_for_test(rw_columns: Vec) -> ConnectorResult { let props = SpecificParserConfig { key_encoding_config: None, encoding_config: EncodingProperties::Json(JsonProperties { use_schema_registry: false, }), - protocol_config: ProtocolProperties::Debezium, + protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()), }; Self::new(props, rw_columns, Default::default()).await } @@ -92,11 +122,12 @@ impl DebeziumParser { key: Option>, payload: Option>, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result { + ) -> ConnectorResult { // tombetone messages are handled implicitly by these accessors - let key_accessor = match key { - None => None, - Some(data) => Some(self.key_builder.generate_accessor(data).await?), + let key_accessor = match (key, self.props.ignore_key) { + (None, false) => None, + (Some(data), false) => Some(self.key_builder.generate_accessor(data).await?), + (_, true) => None, }; let payload_accessor = match payload { None => None, @@ -109,7 +140,7 @@ impl DebeziumParser { Err(err) => { // Only try to access transaction control message if the row operation access failed // to make it a fast path. - if let Ok(transaction_control) = + if let Some(transaction_control) = row_op.transaction_control(&self.source_ctx.connector_props) { Ok(ParseResult::TransactionControl(transaction_control)) @@ -140,7 +171,7 @@ impl ByteStreamSourceParser for DebeziumParser { _key: Option>, _payload: Option>, _writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { + ) -> ConnectorResult<()> { unreachable!("should call `parse_one_with_txn` instead") } @@ -149,7 +180,7 @@ impl ByteStreamSourceParser for DebeziumParser { key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result { + ) -> ConnectorResult { self.parse_inner(key, payload, writer).await } } @@ -186,7 +217,7 @@ mod tests { encoding_config: EncodingProperties::Json(JsonProperties { use_schema_registry: false, }), - protocol_config: ProtocolProperties::Debezium, + protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()), }; let mut source_ctx = SourceContext::default(); source_ctx.connector_props = ConnectorProperties::PostgresCdc(Box::default()); diff --git a/src/connector/src/parser/debezium/mongo_json_parser.rs b/src/connector/src/parser/debezium/mongo_json_parser.rs index 9233bc987eb17..ad0bc9e7f1475 100644 --- a/src/connector/src/parser/debezium/mongo_json_parser.rs +++ b/src/connector/src/parser/debezium/mongo_json_parser.rs @@ -14,17 +14,18 @@ use std::fmt::Debug; -use risingwave_common::error::ErrorCode::{self, ProtocolError}; -use risingwave_common::error::{Result, RwError}; +use anyhow::Context; +use risingwave_common::bail; use risingwave_common::types::DataType; -use simd_json::prelude::MutableObject; -use simd_json::BorrowedValue; -use crate::only_parse_payload; -use crate::parser::unified::debezium::{DebeziumChangeEvent, MongoProjection}; -use crate::parser::unified::json::{JsonAccess, JsonParseOptions}; +use crate::error::ConnectorResult; +use crate::parser::simd_json_parser::DebeziumMongoJsonAccessBuilder; +use crate::parser::unified::debezium::DebeziumChangeEvent; use crate::parser::unified::util::apply_row_operation_on_stream_chunk_writer; -use crate::parser::{ByteStreamSourceParser, ParserFormat, SourceStreamChunkRowWriter}; +use crate::parser::{ + AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, JsonProperties, ParserFormat, + SourceStreamChunkRowWriter, +}; use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef}; #[derive(Debug)] @@ -33,10 +34,24 @@ pub struct DebeziumMongoJsonParser { id_column: SourceColumnDesc, payload_column: SourceColumnDesc, source_ctx: SourceContextRef, + key_builder: AccessBuilderImpl, + payload_builder: AccessBuilderImpl, +} + +fn build_accessor_builder(config: EncodingProperties) -> anyhow::Result { + match config { + EncodingProperties::MongoJson(_) => Ok(AccessBuilderImpl::DebeziumMongoJson( + DebeziumMongoJsonAccessBuilder::new()?, + )), + _ => bail!("unsupported encoding for DEBEZIUM_MONGO format"), + } } impl DebeziumMongoJsonParser { - pub fn new(rw_columns: Vec, source_ctx: SourceContextRef) -> Result { + pub fn new( + rw_columns: Vec, + source_ctx: SourceContextRef, + ) -> ConnectorResult { let id_column = rw_columns .iter() .find(|desc| { @@ -49,57 +64,50 @@ impl DebeziumMongoJsonParser { | DataType::Int64 ) }) - .ok_or_else(|| RwError::from(ProtocolError( - "Debezuim Mongo needs a `_id` column with supported types (Varchar Jsonb int32 int64) in table".into(), - )))?.clone(); + .context("Debezium Mongo needs a `_id` column with supported types (Varchar Jsonb int32 int64) in table")?.clone(); let payload_column = rw_columns .iter() .find(|desc| desc.name == "payload" && matches!(desc.data_type, DataType::Jsonb)) - .ok_or_else(|| { - RwError::from(ProtocolError( - "Debezuim Mongo needs a `payload` column with supported types Jsonb in table" - .into(), - )) - })? + .context("Debezium Mongo needs a `payload` column with supported types Jsonb in table")? .clone(); - if rw_columns.len() != 2 { - return Err(RwError::from(ProtocolError( - "Debezuim Mongo needs no more columns except `_id` and `payload` in table".into(), - ))); + // _rw_{connector}_file/partition & _rw_{connector}_offset are created automatically. + if rw_columns.iter().filter(|desc| desc.is_visible()).count() != 2 { + bail!("Debezium Mongo needs no more columns except `_id` and `payload` in table"); } + // encodings are fixed to MongoJson + let key_builder = + build_accessor_builder(EncodingProperties::MongoJson(JsonProperties::default()))?; + let payload_builder = + build_accessor_builder(EncodingProperties::MongoJson(JsonProperties::default()))?; + Ok(Self { rw_columns, id_column, payload_column, - source_ctx, + key_builder, + payload_builder, }) } - #[allow(clippy::unused_async)] pub async fn parse_inner( - &self, - mut payload: Vec, + &mut self, + key: Option>, + payload: Option>, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result<()> { - let mut event: BorrowedValue<'_> = simd_json::to_borrowed_value(&mut payload) - .map_err(|e| RwError::from(ProtocolError(e.to_string())))?; - - // Event can be configured with and without the "payload" field present. - // See https://github.com/risingwavelabs/risingwave/issues/10178 - - let payload = if let Some(payload) = event.get_mut("payload") { - std::mem::take(payload) - } else { - event + ) -> ConnectorResult<()> { + let key_accessor = match key { + None => None, + Some(data) => Some(self.key_builder.generate_accessor(data).await?), + }; + let payload_accessor = match payload { + None => None, + Some(data) => Some(self.payload_builder.generate_accessor(data).await?), }; - let accessor = JsonAccess::new_with_options(payload, &JsonParseOptions::DEBEZIUM); - - let row_op = DebeziumChangeEvent::with_value(MongoProjection::new(accessor)); - + let row_op = DebeziumChangeEvent::new_mongodb_event(key_accessor, payload_accessor); apply_row_operation_on_stream_chunk_writer(row_op, &mut writer).map_err(Into::into) } } @@ -119,11 +127,11 @@ impl ByteStreamSourceParser for DebeziumMongoJsonParser { async fn parse_one<'a>( &'a mut self, - _key: Option>, + key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { - only_parse_payload!(self, payload, writer) + ) -> ConnectorResult<()> { + self.parse_inner(key, payload, writer).await } } @@ -160,13 +168,40 @@ mod tests { let a = extract_bson_id(&DataType::Varchar, &pld).unwrap(); assert_eq!(a, Some(ScalarImpl::Utf8("5d505646cf6d4fe581014ab2".into()))); } - fn get_columns() -> Vec { - let descs = vec![ - SourceColumnDesc::simple("_id", DataType::Int64, ColumnId::from(0)), + + #[tokio::test] + async fn test_parse_delete_message() { + let (key, payload) = ( + // key + br#"{"schema":null,"payload":{"id":"{\"$oid\": \"65bc9fb6c485f419a7a877fe\"}"}}"#.to_vec(), + // payload + br#"{"schema":null,"payload":{"before":null,"after":null,"updateDescription":null,"source":{"version":"2.4.2.Final","connector":"mongodb","name":"RW_CDC_3001","ts_ms":1706968217000,"snapshot":"false","db":"dev","sequence":null,"rs":"rs0","collection":"test","ord":2,"lsid":null,"txnNumber":null,"wallTime":null},"op":"d","ts_ms":1706968217377,"transaction":null}}"#.to_vec() + ); + + let columns = vec![ + SourceColumnDesc::simple("_id", DataType::Varchar, ColumnId::from(0)), SourceColumnDesc::simple("payload", DataType::Jsonb, ColumnId::from(1)), ]; + let mut parser = DebeziumMongoJsonParser::new(columns.clone(), Default::default()).unwrap(); + let mut builder = SourceStreamChunkBuilder::with_capacity(columns.clone(), 3); + let writer = builder.row_writer(); + parser + .parse_inner(Some(key), Some(payload), writer) + .await + .unwrap(); + let chunk = builder.finish(); + let mut rows = chunk.rows(); + + let (op, row) = rows.next().unwrap(); + assert_eq!(op, Op::Delete); + // oid + assert_eq!( + row.datum_at(0).to_owned_datum(), + (Some(ScalarImpl::Utf8("65bc9fb6c485f419a7a877fe".into()))) + ); - descs + // payload should be null + assert_eq!(row.datum_at(1).to_owned_datum(), None); } #[tokio::test] @@ -177,14 +212,18 @@ mod tests { // data without payload and schema field br#"{"before":null,"after":"{\"_id\": {\"$numberLong\": \"1004\"},\"first_name\": \"Anne\",\"last_name\": \"Kretchmar\",\"email\": \"annek@noanswer.org\"}","patch":null,"filter":null,"updateDescription":null,"source":{"version":"2.1.4.Final","connector":"mongodb","name":"dbserver1","ts_ms":1681879044000,"snapshot":"last","db":"inventory","sequence":null,"rs":"rs0","collection":"customers","ord":1,"lsid":null,"txnNumber":null},"op":"r","ts_ms":1681879054736,"transaction":null}"#.to_vec()]; - let columns = get_columns(); + let columns = vec![ + SourceColumnDesc::simple("_id", DataType::Int64, ColumnId::from(0)), + SourceColumnDesc::simple("payload", DataType::Jsonb, ColumnId::from(1)), + ]; for data in input { - let parser = DebeziumMongoJsonParser::new(columns.clone(), Default::default()).unwrap(); + let mut parser = + DebeziumMongoJsonParser::new(columns.clone(), Default::default()).unwrap(); let mut builder = SourceStreamChunkBuilder::with_capacity(columns.clone(), 3); let writer = builder.row_writer(); - parser.parse_inner(data, writer).await.unwrap(); + parser.parse_inner(None, Some(data), writer).await.unwrap(); let chunk = builder.finish(); let mut rows = chunk.rows(); let (op, row) = rows.next().unwrap(); diff --git a/src/connector/src/parser/debezium/simd_json_parser.rs b/src/connector/src/parser/debezium/simd_json_parser.rs index 5efdd237e9e32..f61545a5443fc 100644 --- a/src/connector/src/parser/debezium/simd_json_parser.rs +++ b/src/connector/src/parser/debezium/simd_json_parser.rs @@ -14,10 +14,12 @@ use std::fmt::Debug; -use risingwave_common::error::{ErrorCode, Result, RwError}; +use anyhow::Context; use simd_json::prelude::MutableObject; use simd_json::BorrowedValue; +use crate::error::ConnectorResult; +use crate::parser::unified::debezium::MongoJsonAccess; use crate::parser::unified::json::{JsonAccess, JsonParseOptions}; use crate::parser::unified::AccessImpl; use crate::parser::AccessBuilder; @@ -28,18 +30,18 @@ pub struct DebeziumJsonAccessBuilder { } impl DebeziumJsonAccessBuilder { - pub fn new() -> Result { + pub fn new() -> ConnectorResult { Ok(Self { value: None }) } } impl AccessBuilder for DebeziumJsonAccessBuilder { #[allow(clippy::unused_async)] - async fn generate_accessor(&mut self, payload: Vec) -> Result> { + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult> { self.value = Some(payload); let mut event: BorrowedValue<'_> = simd_json::to_borrowed_value(self.value.as_mut().unwrap()) - .map_err(|e| RwError::from(ErrorCode::ProtocolError(e.to_string())))?; + .context("failed to parse debezium json payload")?; let payload = if let Some(payload) = event.get_mut("payload") { std::mem::take(payload) @@ -54,6 +56,37 @@ impl AccessBuilder for DebeziumJsonAccessBuilder { } } +#[derive(Debug)] +pub struct DebeziumMongoJsonAccessBuilder { + value: Option>, +} + +impl DebeziumMongoJsonAccessBuilder { + pub fn new() -> anyhow::Result { + Ok(Self { value: None }) + } +} + +impl AccessBuilder for DebeziumMongoJsonAccessBuilder { + #[allow(clippy::unused_async)] + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult> { + self.value = Some(payload); + let mut event: BorrowedValue<'_> = + simd_json::to_borrowed_value(self.value.as_mut().unwrap()) + .context("failed to parse debezium mongo json payload")?; + + let payload = if let Some(payload) = event.get_mut("payload") { + std::mem::take(payload) + } else { + event + }; + + Ok(AccessImpl::MongoJson(MongoJsonAccess::new( + JsonAccess::new_with_options(payload, &JsonParseOptions::DEBEZIUM), + ))) + } +} + #[cfg(test)] mod tests { use std::convert::TryInto; @@ -66,12 +99,14 @@ mod tests { DataType, Date, Interval, Scalar, ScalarImpl, StructType, Time, Timestamp, }; use serde_json::Value; + use thiserror_ext::AsReport; use crate::parser::{ - DebeziumParser, EncodingProperties, JsonProperties, ProtocolProperties, SourceColumnDesc, - SourceStreamChunkBuilder, SpecificParserConfig, + DebeziumParser, DebeziumProps, EncodingProperties, JsonProperties, ProtocolProperties, + SourceColumnDesc, SourceStreamChunkBuilder, SpecificParserConfig, }; use crate::source::SourceContextRef; + fn assert_json_eq(parse_result: &Option, json_str: &str) { if let Some(ScalarImpl::Jsonb(json_val)) = parse_result { let mut json_string = String::new(); @@ -94,7 +129,7 @@ mod tests { encoding_config: EncodingProperties::Json(JsonProperties { use_schema_registry: false, }), - protocol_config: ProtocolProperties::Debezium, + protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()), }; DebeziumParser::new(props, rw_columns, source_ctx) .await @@ -493,7 +528,7 @@ mod tests { } else { // For f64 overflow, the parsing fails let e = res.unwrap_err(); - assert!(e.to_string().contains("InvalidNumber"), "{i}: {e}"); + assert!(e.to_report_string().contains("InvalidNumber"), "{i}: {e}"); } } } @@ -501,7 +536,7 @@ mod tests { // postgres-specific data-type mapping tests mod test3_postgres { - use risingwave_pb::plan_common::AdditionalColumnType; + use risingwave_pb::plan_common::AdditionalColumn; use super::*; use crate::source::SourceColumnType; @@ -566,7 +601,8 @@ mod tests { fields: vec![], column_type: SourceColumnType::Normal, is_pk: false, - additional_column_type: AdditionalColumnType::Normal, + is_hidden_addition_col: false, + additional_column: AdditionalColumn { column_type: None }, }, SourceColumnDesc::simple("o_enum", DataType::Varchar, ColumnId::from(8)), SourceColumnDesc::simple("o_char", DataType::Varchar, ColumnId::from(9)), diff --git a/src/connector/src/parser/json_parser.rs b/src/connector/src/parser/json_parser.rs index 5ed5e2811df98..47db36a0b2b8f 100644 --- a/src/connector/src/parser/json_parser.rs +++ b/src/connector/src/parser/json_parser.rs @@ -14,17 +14,17 @@ use std::collections::HashMap; +use anyhow::Context as _; use apache_avro::Schema; use itertools::{Either, Itertools}; use jst::{convert_avro, Context}; -use risingwave_common::error::ErrorCode::{self, InternalError, ProtocolError}; -use risingwave_common::error::{Result, RwError}; -use risingwave_common::try_match_expand; +use risingwave_common::{bail, try_match_expand}; use risingwave_pb::plan_common::ColumnDesc; use super::avro::schema_resolver::ConfluentSchemaResolver; use super::util::{bytes_from_url, get_kafka_topic}; use super::{EncodingProperties, SchemaRegistryAuth, SpecificParserConfig}; +use crate::error::ConnectorResult; use crate::only_parse_payload; use crate::parser::avro::util::avro_schema_to_column_descs; use crate::parser::unified::json::{JsonAccess, JsonParseOptions}; @@ -44,7 +44,7 @@ pub struct JsonAccessBuilder { impl AccessBuilder for JsonAccessBuilder { #[allow(clippy::unused_async)] - async fn generate_accessor(&mut self, payload: Vec) -> Result> { + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult> { if payload.is_empty() { self.value = Some("{}".into()); } else { @@ -53,7 +53,7 @@ impl AccessBuilder for JsonAccessBuilder { let value = simd_json::to_borrowed_value( &mut self.value.as_mut().unwrap()[self.payload_start_idx..], ) - .map_err(|e| RwError::from(ProtocolError(e.to_string())))?; + .context("failed to parse json payload")?; Ok(AccessImpl::Json(JsonAccess::new_with_options( value, // Debezium and Canal have their special json access builder and will not @@ -64,7 +64,7 @@ impl AccessBuilder for JsonAccessBuilder { } impl JsonAccessBuilder { - pub fn new(use_schema_registry: bool) -> Result { + pub fn new(use_schema_registry: bool) -> ConnectorResult { Ok(Self { value: None, payload_start_idx: if use_schema_registry { 5 } else { 0 }, @@ -86,7 +86,7 @@ impl JsonParser { props: SpecificParserConfig, rw_columns: Vec, source_ctx: SourceContextRef, - ) -> Result { + ) -> ConnectorResult { let json_config = try_match_expand!(props.encoding_config, EncodingProperties::Json)?; let payload_start_idx = if json_config.use_schema_registry { 5 @@ -100,7 +100,7 @@ impl JsonParser { }) } - pub fn new_for_test(rw_columns: Vec) -> Result { + pub fn new_for_test(rw_columns: Vec) -> ConnectorResult { Ok(Self { rw_columns, source_ctx: Default::default(), @@ -113,9 +113,9 @@ impl JsonParser { &self, mut payload: Vec, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result<()> { + ) -> ConnectorResult<()> { let value = simd_json::to_borrowed_value(&mut payload[self.payload_start_idx..]) - .map_err(|e| RwError::from(ProtocolError(e.to_string())))?; + .context("failed to parse json payload")?; let values = if let simd_json::BorrowedValue::Array(arr) = value { Either::Left(arr.into_iter()) } else { @@ -134,11 +134,12 @@ impl JsonParser { if errors.is_empty() { Ok(()) } else { - Err(RwError::from(ErrorCode::InternalError(format!( + // TODO(error-handling): multiple errors + bail!( "failed to parse {} row(s) in a single json message: {}", errors.len(), - errors.iter().join(", ") - )))) + errors.iter().format(", ") + ); } } } @@ -147,7 +148,7 @@ pub async fn schema_to_columns( schema_location: &str, schema_registry_auth: Option, props: &HashMap, -) -> anyhow::Result> { +) -> ConnectorResult> { let url = handle_sr_list(schema_location)?; let json_schema = if let Some(schema_registry_auth) = schema_registry_auth { let client = Client::new(url, &schema_registry_auth)?; @@ -165,8 +166,7 @@ pub async fn schema_to_columns( }; let context = Context::default(); let avro_schema = convert_avro(&json_schema, context).to_string(); - let schema = Schema::parse_str(&avro_schema) - .map_err(|e| RwError::from(InternalError(format!("Avro schema parse error {}", e))))?; + let schema = Schema::parse_str(&avro_schema).context("failed to parse avro schema")?; avro_schema_to_column_descs(&schema) } @@ -188,7 +188,7 @@ impl ByteStreamSourceParser for JsonParser { _key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { + ) -> ConnectorResult<()> { only_parse_payload!(self, payload, writer) } } @@ -203,7 +203,8 @@ mod tests { use risingwave_common::row::Row; use risingwave_common::test_prelude::StreamChunkTestExt; use risingwave_common::types::{DataType, ScalarImpl, ToOwnedDatum}; - use risingwave_pb::plan_common::AdditionalColumnType; + use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; + use risingwave_pb::plan_common::{AdditionalColumn, AdditionalColumnKey}; use super::JsonParser; use crate::parser::upsert_parser::UpsertParser; @@ -580,7 +581,10 @@ mod tests { fields: vec![], column_type: SourceColumnType::Normal, is_pk: true, - additional_column_type: AdditionalColumnType::Key, + is_hidden_addition_col: false, + additional_column: AdditionalColumn { + column_type: Some(AdditionalColumnType::Key(AdditionalColumnKey {})), + }, }; let descs = vec![ SourceColumnDesc::simple("a", DataType::Int32, 0.into()), diff --git a/src/connector/src/parser/maxwell/maxwell_parser.rs b/src/connector/src/parser/maxwell/maxwell_parser.rs index 06c2231e5d104..8ba95ad212130 100644 --- a/src/connector/src/parser/maxwell/maxwell_parser.rs +++ b/src/connector/src/parser/maxwell/maxwell_parser.rs @@ -12,8 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::{ErrorCode, Result, RwError}; +use risingwave_common::bail; +use crate::error::ConnectorResult; use crate::only_parse_payload; use crate::parser::unified::maxwell::MaxwellChangeEvent; use crate::parser::unified::util::apply_row_operation_on_stream_chunk_writer; @@ -35,7 +36,7 @@ impl MaxwellParser { props: SpecificParserConfig, rw_columns: Vec, source_ctx: SourceContextRef, - ) -> Result { + ) -> ConnectorResult { match props.encoding_config { EncodingProperties::Json(_) => { let payload_builder = @@ -47,9 +48,7 @@ impl MaxwellParser { source_ctx, }) } - _ => Err(RwError::from(ErrorCode::ProtocolError( - "unsupported encoding for Maxwell".to_string(), - ))), + _ => bail!("unsupported encoding for Maxwell"), } } @@ -57,7 +56,7 @@ impl MaxwellParser { &mut self, payload: Vec, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result<()> { + ) -> ConnectorResult<()> { let payload_accessor = self.payload_builder.generate_accessor(payload).await?; let row_op = MaxwellChangeEvent::new(payload_accessor); @@ -83,7 +82,7 @@ impl ByteStreamSourceParser for MaxwellParser { _key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { + ) -> ConnectorResult<()> { // restrict the behaviours since there is no corresponding // key/value test for maxwell yet. only_parse_payload!(self, payload, writer) diff --git a/src/connector/src/parser/mod.rs b/src/connector/src/parser/mod.rs index cb82c443ba0a7..810e66b0bef0a 100644 --- a/src/connector/src/parser/mod.rs +++ b/src/connector/src/parser/mod.rs @@ -26,9 +26,8 @@ use futures_async_stream::try_stream; pub use json_parser::*; pub use protobuf::*; use risingwave_common::array::{ArrayBuilderImpl, Op, StreamChunk}; +use risingwave_common::bail; use risingwave_common::catalog::{KAFKA_TIMESTAMP_COLUMN_NAME, TABLE_NAME_COLUMN_NAME}; -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; use risingwave_common::log::LogSuppresser; use risingwave_common::types::{Datum, Scalar, ScalarImpl}; use risingwave_common::util::iter_util::ZipEqFast; @@ -36,7 +35,8 @@ use risingwave_common::util::tracing::InstrumentStream; use risingwave_pb::catalog::{ SchemaRegistryNameStrategy as PbSchemaRegistryNameStrategy, StreamSourceInfo, }; -use risingwave_pb::plan_common::AdditionalColumnType; +use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; +use thiserror_ext::AsReport; use self::avro::AvroAccessBuilder; use self::bytes_parser::BytesAccessBuilder; @@ -44,18 +44,21 @@ pub use self::mysql::mysql_row_to_owned_row; use self::plain_parser::PlainParser; pub use self::postgres::postgres_row_to_owned_row; use self::simd_json_parser::DebeziumJsonAccessBuilder; -use self::unified::{AccessImpl, AccessResult}; +use self::unified::AccessImpl; use self::upsert_parser::UpsertParser; use self::util::get_kafka_topic; use crate::common::AwsAuthProps; +use crate::error::{ConnectorError, ConnectorResult}; use crate::parser::maxwell::MaxwellParser; -use crate::parser::util::{extract_headers_from_meta, extreact_timestamp_from_meta}; +use crate::parser::simd_json_parser::DebeziumMongoJsonAccessBuilder; +use crate::parser::util::{ + extract_header_inner_from_meta, extract_headers_from_meta, extreact_timestamp_from_meta, +}; use crate::schema::schema_registry::SchemaRegistryAuth; use crate::source::monitor::GLOBAL_SOURCE_METRICS; use crate::source::{ - extract_source_struct, BoxSourceStream, SourceColumnDesc, SourceColumnType, SourceContext, - SourceContextRef, SourceEncode, SourceFormat, SourceMeta, SourceWithStateStream, SplitId, - StreamChunkWithState, + extract_source_struct, BoxSourceStream, ChunkSourceStream, SourceColumnDesc, SourceColumnType, + SourceContext, SourceContextRef, SourceEncode, SourceFormat, SourceMessage, SourceMeta, }; pub mod additional_columns; @@ -75,6 +78,9 @@ mod unified; mod upsert_parser; mod util; +pub use debezium::DEBEZIUM_IGNORE_KEY; +pub use unified::{AccessError, AccessResult}; + /// A builder for building a [`StreamChunk`] from [`SourceColumnDesc`]. pub struct SourceStreamChunkBuilder { descs: Vec, @@ -317,7 +323,7 @@ impl SourceStreamChunkRowWriter<'_> { mut f: impl FnMut(&SourceColumnDesc) -> AccessResult, ) -> AccessResult<()> { let mut wrapped_f = |desc: &SourceColumnDesc| { - match (&desc.column_type, &desc.additional_column_type) { + match (&desc.column_type, &desc.additional_column.column_type) { (&SourceColumnType::Offset | &SourceColumnType::RowId, _) => { // SourceColumnType is for CDC source only. Ok(A::output_for( @@ -341,7 +347,7 @@ impl SourceStreamChunkRowWriter<'_> { .unwrap(), // handled all match cases in internal match, unwrap is safe )); } - (_, &AdditionalColumnType::Timestamp) => { + (_, &Some(AdditionalColumnType::Timestamp(_))) => { return Ok(A::output_for( self.row_meta .as_ref() @@ -349,7 +355,7 @@ impl SourceStreamChunkRowWriter<'_> { .unwrap_or(None), )) } - (_, &AdditionalColumnType::Partition) => { + (_, &Some(AdditionalColumnType::Partition(_))) => { // the meta info does not involve spec connector return Ok(A::output_for( self.row_meta @@ -357,7 +363,7 @@ impl SourceStreamChunkRowWriter<'_> { .map(|ele| ScalarImpl::Utf8(ele.split_id.to_string().into())), )); } - (_, &AdditionalColumnType::Offset) => { + (_, &Some(AdditionalColumnType::Offset(_))) => { // the meta info does not involve spec connector return Ok(A::output_for( self.row_meta @@ -365,7 +371,21 @@ impl SourceStreamChunkRowWriter<'_> { .map(|ele| ScalarImpl::Utf8(ele.offset.to_string().into())), )); } - (_, &AdditionalColumnType::Header) => { + (_, &Some(AdditionalColumnType::HeaderInner(ref header_inner))) => { + return Ok(A::output_for( + self.row_meta + .as_ref() + .and_then(|ele| { + extract_header_inner_from_meta( + ele.meta, + header_inner.inner_field.as_ref(), + header_inner.data_type.as_ref(), + ) + }) + .unwrap_or(None), + )) + } + (_, &Some(AdditionalColumnType::Headers(_))) => { return Ok(A::output_for( self.row_meta .as_ref() @@ -373,7 +393,7 @@ impl SourceStreamChunkRowWriter<'_> { .unwrap_or(None), )) } - (_, &AdditionalColumnType::Filename) => { + (_, &Some(AdditionalColumnType::Filename(_))) => { // Filename is used as partition in FS connectors return Ok(A::output_for( self.row_meta @@ -398,7 +418,7 @@ impl SourceStreamChunkRowWriter<'_> { LazyLock::new(LogSuppresser::default); if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::warn!( - %error, + error = %error.as_report(), split_id = self.row_meta.as_ref().map(|m| m.split_id), offset = self.row_meta.as_ref().map(|m| m.offset), column = desc.name, @@ -521,7 +541,7 @@ pub trait ByteStreamSourceParser: Send + Debug + Sized + 'static { key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> impl Future> + Send + 'a; + ) -> impl Future> + Send + 'a; /// Parse one record from the given `payload`, either write rows to the `writer` or interpret it /// as a transaction control message. @@ -535,12 +555,27 @@ pub trait ByteStreamSourceParser: Send + Debug + Sized + 'static { key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> impl Future> + Send + 'a { + ) -> impl Future> + Send + 'a { self.parse_one(key, payload, writer) .map_ok(|_| ParseResult::Rows) } } +#[try_stream(ok = Vec, error = ConnectorError)] +async fn ensure_largest_at_rate_limit(stream: BoxSourceStream, rate_limit: u32) { + #[for_await] + for batch in stream { + let mut batch = batch?; + let mut start = 0; + let end = batch.len(); + while start < end { + let next = std::cmp::min(start + rate_limit as usize, end); + yield std::mem::take(&mut batch[start..next].as_mut()).to_vec(); + start = next; + } + } +} + #[easy_ext::ext(SourceParserIntoStreamExt)] impl P { /// Parse a data stream of one source split into a stream of [`StreamChunk`]. @@ -551,10 +586,16 @@ impl P { /// /// # Returns /// - /// A [`SourceWithStateStream`] which is a stream of parsed messages. - pub fn into_stream(self, data_stream: BoxSourceStream) -> impl SourceWithStateStream { - // Enable tracing to provide more information for parsing failures. - let source_info = self.source_ctx().source_info; + /// A [`ChunkSourceStream`] which is a stream of parsed messages. + pub fn into_stream(self, data_stream: BoxSourceStream) -> impl ChunkSourceStream { + let source_info = self.source_ctx().source_info.clone(); + + // Ensure chunk size is smaller than rate limit + let data_stream = if let Some(rate_limit) = &self.source_ctx().source_ctrl_opts.rate_limit { + Box::pin(ensure_largest_at_rate_limit(data_stream, *rate_limit)) + } else { + data_stream + }; // The parser stream will be long-lived. We use `instrument_with` here to create // a new span for the polling of each chunk. @@ -574,12 +615,11 @@ const MAX_ROWS_FOR_TRANSACTION: usize = 4096; // TODO: when upsert is disabled, how to filter those empty payload // Currently, an err is returned for non upsert with empty payload -#[try_stream(ok = StreamChunkWithState, error = RwError)] +#[try_stream(ok = StreamChunk, error = crate::error::ConnectorError)] async fn into_chunk_stream(mut parser: P, data_stream: BoxSourceStream) { let columns = parser.columns().to_vec(); let mut builder = SourceStreamChunkBuilder::with_capacity(columns, 0); - let mut split_offset_mapping = HashMap::::new(); struct Transaction { id: Box, @@ -604,10 +644,7 @@ async fn into_chunk_stream(mut parser: P, data_stream ); *len = 0; // reset `len` while keeping `id` yield_asap = false; - yield StreamChunkWithState { - chunk: builder.take(batch_len), - split_offset_mapping: Some(std::mem::take(&mut split_offset_mapping)), - }; + yield builder.take(batch_len); } else { // Normal transaction. After the transaction is committed, we should yield the last // batch immediately, so set `yield_asap` to true. @@ -617,7 +654,6 @@ async fn into_chunk_stream(mut parser: P, data_stream // Clean state. Reserve capacity for the builder. assert!(builder.is_empty()); assert!(!yield_asap); - assert!(split_offset_mapping.is_empty()); let _ = builder.take(batch_len); } @@ -625,12 +661,6 @@ async fn into_chunk_stream(mut parser: P, data_stream for (i, msg) in batch.into_iter().enumerate() { if msg.key.is_none() && msg.payload.is_none() { tracing::debug!(offset = msg.offset, "skip parsing of heartbeat message"); - // assumes an empty message as a heartbeat - // heartbeat message offset should not overwrite data messages offset - split_offset_mapping - .entry(msg.split_id) - .or_insert(msg.offset.clone()); - continue; } @@ -644,8 +674,6 @@ async fn into_chunk_stream(mut parser: P, data_stream .observe(lag_ms as f64); } - split_offset_mapping.insert(msg.split_id.clone(), msg.offset.clone()); - let old_op_num = builder.op_num(); match parser .parse_one_with_txn( @@ -677,14 +705,14 @@ async fn into_chunk_stream(mut parser: P, data_stream LazyLock::new(LogSuppresser::default); if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - %error, + error = %error.as_report(), split_id = &*msg.split_id, offset = msg.offset, suppressed_count, "failed to parse message, skipping" ); } - parser.source_ctx().report_user_source_error(error); + parser.source_ctx().report_user_source_error(&error); } } @@ -711,10 +739,7 @@ async fn into_chunk_stream(mut parser: P, data_stream // chunk now. if current_transaction.is_none() && yield_asap { yield_asap = false; - yield StreamChunkWithState { - chunk: builder.take(batch_len - (i + 1)), - split_offset_mapping: Some(std::mem::take(&mut split_offset_mapping)), - }; + yield builder.take(batch_len - (i + 1)); } } } @@ -724,16 +749,13 @@ async fn into_chunk_stream(mut parser: P, data_stream if current_transaction.is_none() { yield_asap = false; - yield StreamChunkWithState { - chunk: builder.take(0), - split_offset_mapping: Some(std::mem::take(&mut split_offset_mapping)), - }; + yield builder.take(0); } } } pub trait AccessBuilder { - async fn generate_accessor(&mut self, payload: Vec) -> Result>; + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult>; } #[derive(Debug)] @@ -750,10 +772,14 @@ pub enum AccessBuilderImpl { Bytes(BytesAccessBuilder), DebeziumAvro(DebeziumAvroAccessBuilder), DebeziumJson(DebeziumJsonAccessBuilder), + DebeziumMongoJson(DebeziumMongoJsonAccessBuilder), } impl AccessBuilderImpl { - pub async fn new_default(config: EncodingProperties, kv: EncodingType) -> Result { + pub async fn new_default( + config: EncodingProperties, + kv: EncodingType, + ) -> ConnectorResult { let accessor = match config { EncodingProperties::Avro(_) => { let config = AvroParserConfig::new(config).await?; @@ -774,7 +800,10 @@ impl AccessBuilderImpl { Ok(accessor) } - pub async fn generate_accessor(&mut self, payload: Vec) -> Result> { + pub async fn generate_accessor( + &mut self, + payload: Vec, + ) -> ConnectorResult> { let accessor = match self { Self::Avro(builder) => builder.generate_accessor(payload).await?, Self::Protobuf(builder) => builder.generate_accessor(payload).await?, @@ -782,6 +811,7 @@ impl AccessBuilderImpl { Self::Bytes(builder) => builder.generate_accessor(payload).await?, Self::DebeziumAvro(builder) => builder.generate_accessor(payload).await?, Self::DebeziumJson(builder) => builder.generate_accessor(payload).await?, + Self::DebeziumMongoJson(builder) => builder.generate_accessor(payload).await?, }; Ok(accessor) } @@ -799,7 +829,7 @@ pub enum ByteStreamSourceParserImpl { CanalJson(CanalJsonParser), } -pub type ParsedStreamImpl = impl SourceWithStateStream + Unpin; +pub type ParsedStreamImpl = impl ChunkSourceStream + Unpin; impl ByteStreamSourceParserImpl { /// Converts this parser into a stream of [`StreamChunk`]. @@ -820,7 +850,10 @@ impl ByteStreamSourceParserImpl { } impl ByteStreamSourceParserImpl { - pub async fn create(parser_config: ParserConfig, source_ctx: SourceContextRef) -> Result { + pub async fn create( + parser_config: ParserConfig, + source_ctx: SourceContextRef, + ) -> ConnectorResult { let CommonParserConfig { rw_columns } = parser_config.common; let protocol = &parser_config.specific.protocol_config; let encode = &parser_config.specific.encoding_config; @@ -845,7 +878,7 @@ impl ByteStreamSourceParserImpl { PlainParser::new(parser_config.specific, rw_columns, source_ctx).await?; Ok(Self::Plain(parser)) } - (ProtocolProperties::Debezium, _) => { + (ProtocolProperties::Debezium(_), _) => { let parser = DebeziumParser::new(parser_config.specific, rw_columns, source_ctx).await?; Ok(Self::Debezium(parser)) @@ -943,28 +976,36 @@ pub enum EncodingProperties { Protobuf(ProtobufProperties), Csv(CsvProperties), Json(JsonProperties), + MongoJson(JsonProperties), Bytes(BytesProperties), Native, + /// Encoding can't be specified because the source will determines it. Now only used in Iceberg. + None, #[default] Unspecified, } #[derive(Debug, Default, Clone)] pub enum ProtocolProperties { - Debezium, + Debezium(DebeziumProps), DebeziumMongo, Maxwell, Canal, Plain, Upsert, Native, + /// Protocol can't be specified because the source will determines it. Now only used in Iceberg. + None, #[default] Unspecified, } impl SpecificParserConfig { // The validity of (format, encode) is ensured by `extract_format_encode` - pub fn new(info: &StreamSourceInfo, with_properties: &HashMap) -> Result { + pub fn new( + info: &StreamSourceInfo, + with_properties: &HashMap, + ) -> ConnectorResult { let source_struct = extract_source_struct(info)?; let format = source_struct.format; let encode = source_struct.encode; @@ -972,7 +1013,11 @@ impl SpecificParserConfig { // in the future let protocol_config = match format { SourceFormat::Native => ProtocolProperties::Native, - SourceFormat::Debezium => ProtocolProperties::Debezium, + SourceFormat::None => ProtocolProperties::None, + SourceFormat::Debezium => { + let debezium_props = DebeziumProps::from(&info.format_encode_options); + ProtocolProperties::Debezium(debezium_props) + } SourceFormat::DebeziumMongo => ProtocolProperties::DebeziumMongo, SourceFormat::Maxwell => ProtocolProperties::Maxwell, SourceFormat::Canal => ProtocolProperties::Canal, @@ -1020,9 +1065,7 @@ impl SpecificParserConfig { (SourceFormat::Plain, SourceEncode::Protobuf) | (SourceFormat::Upsert, SourceEncode::Protobuf) => { if info.row_schema_location.is_empty() { - return Err( - ProtocolError("protobuf file location not provided".to_string()).into(), - ); + bail!("protobuf file location not provided"); } let mut config = ProtobufProperties { message_name: info.proto_message_name.clone(), @@ -1084,11 +1127,9 @@ impl SpecificParserConfig { EncodingProperties::Bytes(BytesProperties { column_name: None }) } (SourceFormat::Native, SourceEncode::Native) => EncodingProperties::Native, + (SourceFormat::None, SourceEncode::None) => EncodingProperties::None, (format, encode) => { - return Err(RwError::from(ProtocolError(format!( - "Unsupported format {:?} encode {:?}", - format, encode - )))); + bail!("Unsupported format {:?} encode {:?}", format, encode); } }; Ok(Self { diff --git a/src/connector/src/parser/mysql.rs b/src/connector/src/parser/mysql.rs index 6e40a6326dc66..5da838c8ff4fb 100644 --- a/src/connector/src/parser/mysql.rs +++ b/src/connector/src/parser/mysql.rs @@ -23,6 +23,7 @@ use risingwave_common::types::{ DataType, Date, Decimal, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz, }; use rust_decimal::Decimal as RustDecimal; +use thiserror_ext::AsReport; static LOG_SUPPERSSER: LazyLock = LazyLock::new(LogSuppresser::default); @@ -32,8 +33,13 @@ macro_rules! handle_data_type { match res { Ok(val) => val.map(|v| ScalarImpl::from(v)), Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { - tracing::error!("parse column `{}` fail: {} ({} suppressed)", $name, err, sc); + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { + tracing::error!( + column = $name, + error = %err.as_report(), + suppressed_count, + "parse column failed", + ); } None } @@ -44,8 +50,13 @@ macro_rules! handle_data_type { match res { Ok(val) => val.map(|v| ScalarImpl::from(<$rw_type>::from(v))), Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { - tracing::error!("parse column `{}` fail: {} ({} suppressed)", $name, err, sc); + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { + tracing::error!( + column = $name, + error = %err.as_report(), + suppressed_count, + "parse column failed", + ); } None } @@ -102,12 +113,12 @@ pub fn mysql_row_to_owned_row(mysql_row: &mut MysqlRow, schema: &Schema) -> Owne ScalarImpl::from(Timestamptz::from_micros(v.timestamp_micros())) }), Err(err) => { - if let Ok(suppressed) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column `{}` fail: {} ({} suppressed)", - name, - err, - suppressed + suppressed_count, + column = name, + error = %err.as_report(), + "parse column failed", ); } None @@ -121,12 +132,12 @@ pub fn mysql_row_to_owned_row(mysql_row: &mut MysqlRow, schema: &Schema) -> Owne match res { Ok(val) => val.map(|v| ScalarImpl::from(v.into_boxed_slice())), Err(err) => { - if let Ok(suppressed) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column `{}` fail: {} ({} suppressed)", - name, - err, - suppressed + suppressed_count, + column = name, + error = %err.as_report(), + "parse column failed", ); } None diff --git a/src/connector/src/parser/plain_parser.rs b/src/connector/src/parser/plain_parser.rs index 1af396db3990a..3b5460de1bfb8 100644 --- a/src/connector/src/parser/plain_parser.rs +++ b/src/connector/src/parser/plain_parser.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; +use risingwave_common::bail; use super::{ AccessBuilderImpl, ByteStreamSourceParser, EncodingProperties, EncodingType, SourceStreamChunkRowWriter, SpecificParserConfig, }; +use crate::error::ConnectorResult; use crate::parser::bytes_parser::BytesAccessBuilder; use crate::parser::simd_json_parser::DebeziumJsonAccessBuilder; use crate::parser::unified::debezium::parse_transaction_meta; @@ -44,7 +44,7 @@ impl PlainParser { props: SpecificParserConfig, rw_columns: Vec, source_ctx: SourceContextRef, - ) -> Result { + ) -> ConnectorResult { let key_builder = if let Some(key_column_name) = get_key_column_name(&rw_columns) { Some(AccessBuilderImpl::Bytes(BytesAccessBuilder::new( EncodingProperties::Bytes(BytesProperties { @@ -62,11 +62,7 @@ impl PlainParser { | EncodingProperties::Bytes(_) => { AccessBuilderImpl::new_default(props.encoding_config, EncodingType::Value).await? } - _ => { - return Err(RwError::from(ProtocolError( - "unsupported encoding for Plain".to_string(), - ))); - } + _ => bail!("Unsupported encoding for Plain"), }; let transaction_meta_builder = Some(AccessBuilderImpl::DebeziumJson( @@ -86,7 +82,7 @@ impl PlainParser { key: Option>, payload: Option>, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result { + ) -> ConnectorResult { // if the message is transaction metadata, parse it and return if let Some(msg_meta) = writer.row_meta && let SourceMeta::DebeziumCdc(cdc_meta) = msg_meta.meta @@ -150,7 +146,7 @@ impl ByteStreamSourceParser for PlainParser { _key: Option>, _payload: Option>, _writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { + ) -> ConnectorResult<()> { unreachable!("should call `parse_one_with_txn` instead") } @@ -159,7 +155,7 @@ impl ByteStreamSourceParser for PlainParser { key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result { + ) -> ConnectorResult { self.parse_inner(key, payload, writer).await } } @@ -218,18 +214,18 @@ mod tests { let output = output .unwrap() .into_iter() - .filter(|c| c.chunk.cardinality() > 0) + .filter(|c| c.cardinality() > 0) .enumerate() .map(|(i, c)| { if i == 0 { // begin + 3 data messages - assert_eq!(4, c.chunk.cardinality()); + assert_eq!(4, c.cardinality()); } if i == 1 { // 2 data messages + 1 end - assert_eq!(3, c.chunk.cardinality()); + assert_eq!(3, c.cardinality()); } - c.chunk + c }) .collect_vec(); @@ -255,11 +251,11 @@ mod tests { let output = output .unwrap() .into_iter() - .filter(|c| c.chunk.cardinality() > 0) + .filter(|c| c.cardinality() > 0) .map(|c| { // 5 data messages in a single chunk - assert_eq!(5, c.chunk.cardinality()); - c.chunk + assert_eq!(5, c.cardinality()); + c }) .collect_vec(); @@ -267,11 +263,11 @@ mod tests { assert_eq!(1, output.len()); } - #[try_stream(ok = Vec, error = anyhow::Error)] + #[try_stream(ok = Vec, error = crate::error::ConnectorError)] async fn source_message_stream(transactional: bool) { let begin_msg = r#"{"schema":null,"payload":{"status":"BEGIN","id":"35352:3962948040","event_count":null,"data_collections":null,"ts_ms":1704269323180}}"#; let commit_msg = r#"{"schema":null,"payload":{"status":"END","id":"35352:3962950064","event_count":11,"data_collections":[{"data_collection":"public.orders_tx","event_count":5},{"data_collection":"public.person","event_count":6}],"ts_ms":1704269323180}}"#; - let data_batches = vec![ + let data_batches = [ vec![ r#"{ "schema": null, "payload": {"after": {"customer_name": "a1", "order_date": "2020-01-30", "order_id": 10021, "order_status": false, "price": "50.50", "product_id": 102}, "before": null, "op": "c", "source": {"connector": "postgresql", "db": "mydb", "lsn": 3963199336, "name": "RW_CDC_1001", "schema": "public", "sequence": "[\"3963198512\",\"3963199336\"]", "snapshot": "false", "table": "orders_tx", "ts_ms": 1704355505506, "txId": 35352, "version": "2.4.2.Final", "xmin": null}, "transaction": {"data_collection_order": 1, "id": "35392:3963199336", "total_order": 1}, "ts_ms": 1704355839905} }"#, r#"{ "schema": null, "payload": {"after": {"customer_name": "a2", "order_date": "2020-02-30", "order_id": 10022, "order_status": false, "price": "50.50", "product_id": 102}, "before": null, "op": "c", "source": {"connector": "postgresql", "db": "mydb", "lsn": 3963199336, "name": "RW_CDC_1001", "schema": "public", "sequence": "[\"3963198512\",\"3963199336\"]", "snapshot": "false", "table": "orders_tx", "ts_ms": 1704355505506, "txId": 35352, "version": "2.4.2.Final", "xmin": null}, "transaction": {"data_collection_order": 1, "id": "35392:3963199336", "total_order": 1}, "ts_ms": 1704355839905} }"#, diff --git a/src/connector/src/parser/postgres.rs b/src/connector/src/parser/postgres.rs index 0823fa5579557..f2c541423eb71 100644 --- a/src/connector/src/parser/postgres.rs +++ b/src/connector/src/parser/postgres.rs @@ -23,6 +23,8 @@ use risingwave_common::types::{ Timestamptz, }; use rust_decimal::Decimal as RustDecimal; +use thiserror_ext::AsReport; +use tokio_postgres::types::Type; static LOG_SUPPERSSER: LazyLock = LazyLock::new(LogSuppresser::default); @@ -37,12 +39,12 @@ macro_rules! handle_list_data_type { } } Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column \"{}\" fail: {} ({} suppressed)", - $name, - err, - sc + column = $name, + error = %err.as_report(), + suppressed_count, + "parse column failed", ); } } @@ -59,12 +61,12 @@ macro_rules! handle_list_data_type { } } Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column \"{}\" fail: {} ({} suppressed)", - $name, - err, - sc + column = $name, + error = %err.as_report(), + suppressed_count, + "parse column failed", ); } } @@ -78,12 +80,12 @@ macro_rules! handle_data_type { match res { Ok(val) => val.map(|v| ScalarImpl::from(v)), Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column \"{}\" fail: {} ({} suppressed)", - $name, - err, - sc + column = $name, + error = %err.as_report(), + suppressed_count, + "parse column failed", ); } None @@ -95,12 +97,12 @@ macro_rules! handle_data_type { match res { Ok(val) => val.map(|v| ScalarImpl::from(<$rw_type>::from(v))), Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column \"{}\" fail: {} ({} suppressed)", - $name, - err, - sc + column = $name, + error = %err.as_report(), + suppressed_count, + "parse column failed", ); } None @@ -138,7 +140,29 @@ pub fn postgres_row_to_owned_row(row: tokio_postgres::Row, schema: &Schema) -> O handle_data_type!(row, i, name, RustDecimal, Decimal) } DataType::Varchar => { - handle_data_type!(row, i, name, String) + match row.columns()[i].type_() { + // Since we don't support UUID natively, adapt it to a VARCHAR column + &Type::UUID => { + let res = row.try_get::<_, Option>(i); + match res { + Ok(val) => val.map(|v| ScalarImpl::from(v.to_string())), + Err(err) => { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { + tracing::error!( + suppressed_count, + column = name, + error = %err.as_report(), + "parse uuid column failed", + ); + } + None + } + } + } + _ => { + handle_data_type!(row, i, name, String) + } + } } DataType::Date => { handle_data_type!(row, i, name, NaiveDate, Date) @@ -157,12 +181,12 @@ pub fn postgres_row_to_owned_row(row: tokio_postgres::Row, schema: &Schema) -> O match res { Ok(val) => val.map(|v| ScalarImpl::from(v.into_boxed_slice())), Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column \"{}\" fail: {} ({} suppressed)", - name, - err, - sc + suppressed_count, + column = name, + error = %err.as_report(), + "parse column failed", ); } None @@ -254,12 +278,12 @@ pub fn postgres_row_to_owned_row(row: tokio_postgres::Row, schema: &Schema) -> O } } Err(err) => { - if let Ok(sc) = LOG_SUPPERSSER.check() { + if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { tracing::error!( - "parse column \"{}\" fail: {} ({} suppressed)", - name, - err, - sc + suppressed_count, + column = name, + error = %err.as_report(), + "parse column failed", ); } } diff --git a/src/connector/src/parser/protobuf/parser.rs b/src/connector/src/parser/protobuf/parser.rs index a278179924dda..b09cc24ea59e1 100644 --- a/src/connector/src/parser/protobuf/parser.rs +++ b/src/connector/src/parser/protobuf/parser.rs @@ -14,25 +14,29 @@ use std::sync::Arc; +use anyhow::Context; use itertools::Itertools; use prost_reflect::{ Cardinality, DescriptorPool, DynamicMessage, FieldDescriptor, Kind, MessageDescriptor, ReflectMessage, Value, }; use risingwave_common::array::{ListValue, StructValue}; -use risingwave_common::error::ErrorCode::{InternalError, ProtocolError}; -use risingwave_common::error::{Result, RwError}; -use risingwave_common::try_match_expand; use risingwave_common::types::{DataType, Datum, Decimal, JsonbVal, ScalarImpl, F32, F64}; -use risingwave_pb::plan_common::{AdditionalColumnType, ColumnDesc, ColumnDescVersion}; +use risingwave_common::{bail, try_match_expand}; +use risingwave_pb::plan_common::{AdditionalColumn, ColumnDesc, ColumnDescVersion}; +use thiserror::Error; +use thiserror_ext::{AsReport, Macro}; use super::schema_resolver::*; +use crate::error::ConnectorResult; use crate::parser::unified::protobuf::ProtobufAccess; -use crate::parser::unified::AccessImpl; +use crate::parser::unified::{ + bail_uncategorized, uncategorized, AccessError, AccessImpl, AccessResult, +}; use crate::parser::util::bytes_from_url; use crate::parser::{AccessBuilder, EncodingProperties}; use crate::schema::schema_registry::{ - extract_schema_id, get_subject_by_strategy, handle_sr_list, Client, + extract_schema_id, get_subject_by_strategy, handle_sr_list, Client, WireFormatError, }; #[derive(Debug)] @@ -44,7 +48,7 @@ pub struct ProtobufAccessBuilder { impl AccessBuilder for ProtobufAccessBuilder { #[allow(clippy::unused_async)] - async fn generate_accessor(&mut self, payload: Vec) -> Result> { + async fn generate_accessor(&mut self, payload: Vec) -> ConnectorResult> { let payload = if self.confluent_wire_type { resolve_pb_header(&payload)? } else { @@ -52,7 +56,7 @@ impl AccessBuilder for ProtobufAccessBuilder { }; let message = DynamicMessage::decode(self.message_descriptor.clone(), payload) - .map_err(|e| ProtocolError(format!("parse message failed: {}", e)))?; + .context("failed to parse message")?; Ok(AccessImpl::Protobuf(ProtobufAccess::new( message, @@ -62,7 +66,7 @@ impl AccessBuilder for ProtobufAccessBuilder { } impl ProtobufAccessBuilder { - pub fn new(config: ProtobufParserConfig) -> Result { + pub fn new(config: ProtobufParserConfig) -> ConnectorResult { let ProtobufParserConfig { confluent_wire_type, message_descriptor, @@ -86,17 +90,15 @@ pub struct ProtobufParserConfig { } impl ProtobufParserConfig { - pub async fn new(encoding_properties: EncodingProperties) -> Result { + pub async fn new(encoding_properties: EncodingProperties) -> ConnectorResult { let protobuf_config = try_match_expand!(encoding_properties, EncodingProperties::Protobuf)?; let location = &protobuf_config.row_schema_location; let message_name = &protobuf_config.message_name; let url = handle_sr_list(location.as_str())?; - if let Some(name) = protobuf_config.key_message_name { + if protobuf_config.key_message_name.is_some() { // https://docs.confluent.io/platform/7.5/control-center/topics/schema.html#c3-schemas-best-practices-key-value-pairs - return Err(RwError::from(ProtocolError(format!( - "key.message = {name} not used. Protobuf key unsupported." - )))); + bail!("protobuf key is not supported"); } let schema_bytes = if protobuf_config.use_schema_registry { let schema_value = get_subject_by_strategy( @@ -114,18 +116,14 @@ impl ProtobufParserConfig { bytes_from_url(url, protobuf_config.aws_auth_props.as_ref()).await? }; - let pool = DescriptorPool::decode(schema_bytes.as_slice()).map_err(|e| { - ProtocolError(format!( - "cannot build descriptor pool from schema: {}, error: {}", - location, e - )) - })?; + let pool = DescriptorPool::decode(schema_bytes.as_slice()) + .with_context(|| format!("cannot build descriptor pool from schema `{}`", location))?; - let message_descriptor = pool.get_message_by_name(message_name).ok_or_else(|| { - ProtocolError(format!( - "Cannot find message {} in schema: {}.\nDescriptor pool is {:?}", - message_name, location, pool - )) + let message_descriptor = pool.get_message_by_name(message_name).with_context(|| { + format!( + "cannot find message `{}` in schema `{}`", + message_name, location, + ) })?; Ok(Self { @@ -136,7 +134,7 @@ impl ProtobufParserConfig { } /// Maps the protobuf schema to relational schema. - pub fn map_to_columns(&self) -> Result> { + pub fn map_to_columns(&self) -> ConnectorResult> { let mut columns = Vec::with_capacity(self.message_descriptor.fields().len()); let mut index = 0; let mut parse_trace: Vec = vec![]; @@ -156,15 +154,16 @@ impl ProtobufParserConfig { field_descriptor: &FieldDescriptor, index: &mut i32, parse_trace: &mut Vec, - ) -> Result { - let field_type = protobuf_type_mapping(field_descriptor, parse_trace)?; + ) -> ConnectorResult { + let field_type = protobuf_type_mapping(field_descriptor, parse_trace) + .context("failed to map protobuf type")?; if let Kind::Message(m) = field_descriptor.kind() { let field_descs = if let DataType::List { .. } = field_type { vec![] } else { m.fields() .map(|f| Self::pb_field_to_col_desc(&f, index, parse_trace)) - .collect::>>()? + .try_collect()? }; *index += 1; Ok(ColumnDesc { @@ -175,7 +174,8 @@ impl ProtobufParserConfig { type_name: m.full_name().to_string(), generated_or_default_column: None, description: None, - additional_column_type: AdditionalColumnType::Normal as i32, + additional_column_type: 0, // deprecated + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, }) } else { @@ -184,7 +184,7 @@ impl ProtobufParserConfig { column_id: *index, name: field_descriptor.name().to_string(), column_type: Some(field_type.to_protobuf()), - additional_column_type: AdditionalColumnType::Normal as i32, + additional_column: Some(AdditionalColumn { column_type: None }), version: ColumnDescVersion::Pr13707 as i32, ..Default::default() }) @@ -192,15 +192,22 @@ impl ProtobufParserConfig { } } -fn detect_loop_and_push(trace: &mut Vec, fd: &FieldDescriptor) -> Result<()> { +#[derive(Error, Debug, Macro)] +#[error("{0}")] +struct ProtobufTypeError(#[message] String); + +fn detect_loop_and_push( + trace: &mut Vec, + fd: &FieldDescriptor, +) -> std::result::Result<(), ProtobufTypeError> { let identifier = format!("{}({})", fd.name(), fd.full_name()); if trace.iter().any(|s| s == identifier.as_str()) { - return Err(RwError::from(ProtocolError(format!( + bail_protobuf_type_error!( "circular reference detected: {}, conflict with {}, kind {:?}", - trace.iter().join("->"), + trace.iter().format("->"), identifier, fd.kind(), - )))); + ); } trace.push(identifier); Ok(()) @@ -341,7 +348,9 @@ pub fn from_protobuf_value( field_desc: &FieldDescriptor, value: &Value, descriptor_pool: &Arc, -) -> Result { +) -> AccessResult { + let kind = field_desc.kind(); + let v = match value { Value::Bool(v) => ScalarImpl::Bool(*v), Value::I32(i) => ScalarImpl::Int32(*i), @@ -352,17 +361,13 @@ pub fn from_protobuf_value( Value::F64(f) => ScalarImpl::Float64(F64::from(*f)), Value::String(s) => ScalarImpl::Utf8(s.as_str().into()), Value::EnumNumber(idx) => { - let kind = field_desc.kind(); - let enum_desc = kind.as_enum().ok_or_else(|| { - let err_msg = format!("protobuf parse error.not a enum desc {:?}", field_desc); - RwError::from(ProtocolError(err_msg)) + let enum_desc = kind.as_enum().ok_or_else(|| AccessError::TypeError { + expected: "enum".to_owned(), + got: format!("{kind:?}"), + value: value.to_string(), })?; let enum_symbol = enum_desc.get_value(*idx).ok_or_else(|| { - let err_msg = format!( - "protobuf parse error.unknown enum index {} of enum {:?}", - idx, enum_desc - ); - RwError::from(ProtocolError(err_msg)) + uncategorized!("unknown enum index {} of enum {:?}", idx, enum_desc) })?; ScalarImpl::Utf8(enum_symbol.name().into()) } @@ -389,18 +394,14 @@ pub fn from_protobuf_value( let Some(ScalarImpl::Bytea(payload)) = from_protobuf_value(&payload_field_desc, &payload, descriptor_pool)? else { - let err_msg = "Expected ScalarImpl::Bytea for payload".to_string(); - return Err(RwError::from(ProtocolError(err_msg))); + bail_uncategorized!("expected bytes for dynamic message payload"); }; // Get the corresponding schema from the descriptor pool let msg_desc = descriptor_pool .get_message_by_name(&type_url) .ok_or_else(|| { - ProtocolError(format!( - "Cannot find message {} in from_protobuf_value.\nDescriptor pool is {:#?}", - type_url, descriptor_pool - )) + uncategorized!("message `{type_url}` not found in descriptor pool") })?; let f = msg_desc @@ -439,11 +440,10 @@ pub fn from_protobuf_value( if !dyn_msg.has_field(&field_desc) && field_desc.cardinality() == Cardinality::Required { - let err_msg = format!( - "protobuf parse error.missing required field {:?}", - field_desc - ); - return Err(RwError::from(ProtocolError(err_msg))); + return Err(AccessError::Undefined { + name: field_desc.name().to_owned(), + path: dyn_msg.descriptor().full_name().to_owned(), + }); } // use default value if dyn_msg doesn't has this field let value = dyn_msg.get_field(&field_desc); @@ -453,7 +453,8 @@ pub fn from_protobuf_value( } } Value::List(values) => { - let data_type = protobuf_type_mapping(field_desc, &mut vec![])?; + let data_type = protobuf_type_mapping(field_desc, &mut vec![]) + .map_err(|e| uncategorized!("{}", e.to_report_string()))?; let mut builder = data_type.as_list().create_array_builder(values.len()); for value in values { builder.append(from_protobuf_value(field_desc, value, descriptor_pool)?); @@ -462,11 +463,9 @@ pub fn from_protobuf_value( } Value::Bytes(value) => ScalarImpl::Bytea(value.to_vec().into_boxed_slice()), _ => { - let err_msg = format!( - "protobuf parse error.unsupported type {:?}, value {:?}", - field_desc, value - ); - return Err(RwError::from(InternalError(err_msg))); + return Err(AccessError::UnsupportedType { + ty: format!("{kind:?}"), + }); } }; Ok(Some(v)) @@ -476,7 +475,7 @@ pub fn from_protobuf_value( fn protobuf_type_mapping( field_descriptor: &FieldDescriptor, parse_trace: &mut Vec, -) -> Result { +) -> std::result::Result { detect_loop_and_push(parse_trace, field_descriptor)?; let field_type = field_descriptor.kind(); let mut t = match field_type { @@ -494,7 +493,7 @@ fn protobuf_type_mapping( let fields = m .fields() .map(|f| protobuf_type_mapping(&f, parse_trace)) - .collect::>>()?; + .try_collect()?; let field_names = m.fields().map(|f| f.name().to_string()).collect_vec(); // Note that this part is useful for actual parsing @@ -513,10 +512,10 @@ fn protobuf_type_mapping( Kind::Bytes => DataType::Bytea, }; if field_descriptor.is_map() { - return Err(RwError::from(ProtocolError(format!( - "map type is unsupported (field: '{}')", + bail_protobuf_type_error!( + "protobuf map type (on field `{}`) is not supported", field_descriptor.full_name() - )))); + ); } if field_descriptor.cardinality() == Cardinality::Repeated { t = DataType::List(Box::new(t)) @@ -525,19 +524,56 @@ fn protobuf_type_mapping( Ok(t) } -pub(crate) fn resolve_pb_header(payload: &[u8]) -> Result<&[u8]> { +/// A port from the implementation of confluent's Varint Zig-zag deserialization. +/// See `ReadVarint` in +fn decode_varint_zigzag(buffer: &[u8]) -> ConnectorResult<(i32, usize)> { + // We expect the decoded number to be 4 bytes. + let mut value = 0u32; + let mut shift = 0; + let mut len = 0usize; + + for &byte in buffer { + len += 1; + // The Varint encoding is limited to 5 bytes. + if len > 5 { + break; + } + // The byte is cast to u32 to avoid shifting overflow. + let byte_ext = byte as u32; + // In Varint encoding, the lowest 7 bits are used to represent number, + // while the highest zero bit indicates the end of the number with Varint encoding. + value |= (byte_ext & 0x7F) << shift; + if byte_ext & 0x80 == 0 { + return Ok((((value >> 1) as i32) ^ -((value & 1) as i32), len)); + } + + shift += 7; + } + + Err(WireFormatError::ParseMessageIndexes.into()) +} + +/// Reference: +/// Wire format for Confluent pb header is: +/// | 0 | 1-4 | 5-x | x+1-end +/// | magic-byte | schema-id | message-indexes | protobuf-payload +pub(crate) fn resolve_pb_header(payload: &[u8]) -> ConnectorResult<&[u8]> { // there's a message index array at the front of payload // if it is the first message in proto def, the array is just and `0` - // TODO: support parsing more complex index array let (_, remained) = extract_schema_id(payload)?; + // The message indexes are encoded as int using variable-length zig-zag encoding, + // prefixed by the length of the array. + // Note that if the first byte is 0, it is equivalent to (1, 0) as an optimization. match remained.first() { Some(0) => Ok(&remained[1..]), - Some(i) => { - Err(RwError::from(ProtocolError(format!("The payload message must be the first message in protobuf schema def, but the message index is {}", i)))) - } - None => { - Err(RwError::from(ProtocolError("The proto payload is empty".to_owned()))) + Some(_) => { + let (index_len, mut offset) = decode_varint_zigzag(remained)?; + for _ in 0..index_len { + offset += decode_varint_zigzag(&remained[offset..])?.1; + } + Ok(&remained[offset..]) } + None => bail!("The proto payload is empty"), } } @@ -576,7 +612,7 @@ mod test { static PRE_GEN_PROTO_DATA: &[u8] = b"\x08\x7b\x12\x0c\x74\x65\x73\x74\x20\x61\x64\x64\x72\x65\x73\x73\x1a\x09\x74\x65\x73\x74\x20\x63\x69\x74\x79\x20\xc8\x03\x2d\x19\x04\x9e\x3f\x32\x0a\x32\x30\x32\x31\x2d\x30\x31\x2d\x30\x31"; #[tokio::test] - async fn test_simple_schema() -> Result<()> { + async fn test_simple_schema() -> crate::error::ConnectorResult<()> { let location = schema_dir() + "/simple-schema"; println!("location: {}", location); let message_name = "test.TestRecord"; @@ -621,7 +657,7 @@ mod test { } #[tokio::test] - async fn test_complex_schema() -> Result<()> { + async fn test_complex_schema() -> crate::error::ConnectorResult<()> { let location = schema_dir() + "/complex-schema"; let message_name = "test.User"; @@ -913,7 +949,7 @@ mod test { static ANY_GEN_PROTO_DATA: &[u8] = b"\x08\xb9\x60\x12\x32\x0a\x24\x74\x79\x70\x65\x2e\x67\x6f\x6f\x67\x6c\x65\x61\x70\x69\x73\x2e\x63\x6f\x6d\x2f\x74\x65\x73\x74\x2e\x53\x74\x72\x69\x6e\x67\x56\x61\x6c\x75\x65\x12\x0a\x0a\x08\x4a\x6f\x68\x6e\x20\x44\x6f\x65"; #[tokio::test] - async fn test_any_schema() -> Result<()> { + async fn test_any_schema() -> crate::error::ConnectorResult<()> { let conf = create_recursive_pb_parser_config("/any-schema.pb", "test.TestAny").await; println!("Current conf: {:#?}", conf); @@ -974,7 +1010,7 @@ mod test { static ANY_GEN_PROTO_DATA_1: &[u8] = b"\x08\xb9\x60\x12\x2b\x0a\x23\x74\x79\x70\x65\x2e\x67\x6f\x6f\x67\x6c\x65\x61\x70\x69\x73\x2e\x63\x6f\x6d\x2f\x74\x65\x73\x74\x2e\x49\x6e\x74\x33\x32\x56\x61\x6c\x75\x65\x12\x04\x08\xd2\xfe\x06"; #[tokio::test] - async fn test_any_schema_1() -> Result<()> { + async fn test_any_schema_1() -> crate::error::ConnectorResult<()> { let conf = create_recursive_pb_parser_config("/any-schema.pb", "test.TestAny").await; println!("Current conf: {:#?}", conf); @@ -1043,7 +1079,7 @@ mod test { static ANY_RECURSIVE_GEN_PROTO_DATA: &[u8] = b"\x08\xb9\x60\x12\x84\x01\x0a\x21\x74\x79\x70\x65\x2e\x67\x6f\x6f\x67\x6c\x65\x61\x70\x69\x73\x2e\x63\x6f\x6d\x2f\x74\x65\x73\x74\x2e\x41\x6e\x79\x56\x61\x6c\x75\x65\x12\x5f\x0a\x30\x0a\x24\x74\x79\x70\x65\x2e\x67\x6f\x6f\x67\x6c\x65\x61\x70\x69\x73\x2e\x63\x6f\x6d\x2f\x74\x65\x73\x74\x2e\x53\x74\x72\x69\x6e\x67\x56\x61\x6c\x75\x65\x12\x08\x0a\x06\x31\x31\x34\x35\x31\x34\x12\x2b\x0a\x23\x74\x79\x70\x65\x2e\x67\x6f\x6f\x67\x6c\x65\x61\x70\x69\x73\x2e\x63\x6f\x6d\x2f\x74\x65\x73\x74\x2e\x49\x6e\x74\x33\x32\x56\x61\x6c\x75\x65\x12\x04\x08\xd2\xfe\x06"; #[tokio::test] - async fn test_any_recursive() -> Result<()> { + async fn test_any_recursive() -> crate::error::ConnectorResult<()> { let conf = create_recursive_pb_parser_config("/any-schema.pb", "test.TestAny").await; println!("Current conf: {:#?}", conf); @@ -1104,4 +1140,54 @@ mod test { Ok(()) } + + #[test] + fn test_decode_varint_zigzag() { + // 1. Positive number + let buffer = vec![0x02]; + let (value, len) = decode_varint_zigzag(&buffer).unwrap(); + assert_eq!(value, 1); + assert_eq!(len, 1); + + // 2. Negative number + let buffer = vec![0x01]; + let (value, len) = decode_varint_zigzag(&buffer).unwrap(); + assert_eq!(value, -1); + assert_eq!(len, 1); + + // 3. Larger positive number + let buffer = vec![0x9E, 0x03]; + let (value, len) = decode_varint_zigzag(&buffer).unwrap(); + assert_eq!(value, 207); + assert_eq!(len, 2); + + // 4. Larger negative number + let buffer = vec![0xBF, 0x07]; + let (value, len) = decode_varint_zigzag(&buffer).unwrap(); + assert_eq!(value, -480); + assert_eq!(len, 2); + + // 5. Maximum positive number + let buffer = vec![0xFE, 0xFF, 0xFF, 0xFF, 0x0F]; + let (value, len) = decode_varint_zigzag(&buffer).unwrap(); + assert_eq!(value, i32::MAX); + assert_eq!(len, 5); + + // 6. Maximum negative number + let buffer = vec![0xFF, 0xFF, 0xFF, 0xFF, 0x0F]; + let (value, len) = decode_varint_zigzag(&buffer).unwrap(); + assert_eq!(value, i32::MIN); + assert_eq!(len, 5); + + // 7. More than 32 bits + let buffer = vec![0xFF, 0xFF, 0xFF, 0xFF, 0x7F]; + let (value, len) = decode_varint_zigzag(&buffer).unwrap(); + assert_eq!(value, i32::MIN); + assert_eq!(len, 5); + + // 8. Invalid input (more than 5 bytes) + let buffer = vec![0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF]; + let result = decode_varint_zigzag(&buffer); + assert!(result.is_err()); + } } diff --git a/src/connector/src/parser/protobuf/schema_resolver.rs b/src/connector/src/parser/protobuf/schema_resolver.rs index 47720ae13b9f6..828843842c785 100644 --- a/src/connector/src/parser/protobuf/schema_resolver.rs +++ b/src/connector/src/parser/protobuf/schema_resolver.rs @@ -15,14 +15,14 @@ use std::iter; use std::path::Path; +use anyhow::Context; use itertools::Itertools; use protobuf_native::compiler::{ SimpleErrorCollector, SourceTreeDescriptorDatabase, VirtualSourceTree, }; use protobuf_native::MessageLite; -use risingwave_common::error::ErrorCode::{InternalError, ProtocolError}; -use risingwave_common::error::{Result, RwError}; +use crate::error::ConnectorResult; use crate::schema::schema_registry::Client; macro_rules! embed_wkts { @@ -55,9 +55,11 @@ const WELL_KNOWN_TYPES: &[(&str, &[u8])] = embed_wkts![ pub(super) async fn compile_file_descriptor_from_schema_registry( subject_name: &str, client: &Client, -) -> Result> { - let (primary_subject, dependency_subjects) = - client.get_subject_and_references(subject_name).await?; +) -> ConnectorResult> { + let (primary_subject, dependency_subjects) = client + .get_subject_and_references(subject_name) + .await + .with_context(|| format!("failed to resolve subject `{subject_name}`"))?; // Compile .proto files into a file descriptor set. let mut source_tree = VirtualSourceTree::new(); @@ -81,12 +83,13 @@ pub(super) async fn compile_file_descriptor_from_schema_registry( db.as_mut() .build_file_descriptor_set(&[Path::new(&primary_subject.name)]) } - .map_err(|_| { - RwError::from(ProtocolError(format!( + .with_context(|| { + format!( "build_file_descriptor_set failed. Errors:\n{}", error_collector.as_mut().join("\n") - ))) + ) })?; - fds.serialize() - .map_err(|_| RwError::from(InternalError("serialize descriptor set failed".to_owned()))) + + let serialized = fds.serialize().context("serialize descriptor set failed")?; + Ok(serialized) } diff --git a/src/connector/src/parser/unified/avro.rs b/src/connector/src/parser/unified/avro.rs index 6b34a953f3b11..f0825d9af4bbe 100644 --- a/src/connector/src/parser/unified/avro.rs +++ b/src/connector/src/parser/unified/avro.rs @@ -15,7 +15,6 @@ use std::str::FromStr; use std::sync::LazyLock; -use anyhow::anyhow; use apache_avro::schema::{DecimalSchema, RecordSchema}; use apache_avro::types::Value; use apache_avro::{Decimal as AvroDecimal, Schema}; @@ -23,13 +22,15 @@ use chrono::Datelike; use itertools::Itertools; use num_bigint::{BigInt, Sign}; use risingwave_common::array::{ListValue, StructValue}; -use risingwave_common::cast::{i64_to_timestamp, i64_to_timestamptz}; -use risingwave_common::error::Result as RwResult; +use risingwave_common::bail; use risingwave_common::log::LogSuppresser; -use risingwave_common::types::{DataType, Date, Datum, Interval, JsonbVal, ScalarImpl, Time}; +use risingwave_common::types::{ + DataType, Date, Datum, Interval, JsonbVal, ScalarImpl, Time, Timestamp, Timestamptz, +}; use risingwave_common::util::iter_util::ZipEqFast; -use super::{Access, AccessError, AccessResult}; +use super::{bail_uncategorized, uncategorized, Access, AccessError, AccessResult}; +use crate::error::ConnectorResult; #[derive(Clone)] /// Options for parsing an `AvroValue` into Datum, with an optional avro schema. pub struct AvroParseOptions<'a> { @@ -135,29 +136,28 @@ impl<'a> AvroParseOptions<'a> { .iter() .find(|field| field.0 == field_name) .map(|field| &field.1) + .ok_or_else(|| { + uncategorized!("`{field_name}` field not found in VariableScaleDecimal") + }) }; - let scale = match find_in_records("scale").ok_or_else(|| { - AccessError::Other(anyhow!("scale field not found in VariableScaleDecimal")) - })? { - Value::Int(scale) => Ok(*scale), - avro_value => Err(AccessError::Other(anyhow!( + let scale = match find_in_records("scale")? { + Value::Int(scale) => *scale, + avro_value => bail_uncategorized!( "scale field in VariableScaleDecimal is not int, got {:?}", avro_value - ))), - }?; - - let value: BigInt = match find_in_records("value").ok_or_else(|| { - AccessError::Other(anyhow!("value field not found in VariableScaleDecimal")) - })? { - Value::Bytes(bytes) => Ok(BigInt::from_signed_bytes_be(bytes)), - avro_value => Err(AccessError::Other(anyhow!( + ), + }; + + let value: BigInt = match find_in_records("value")? { + Value::Bytes(bytes) => BigInt::from_signed_bytes_be(bytes), + avro_value => bail_uncategorized!( "value field in VariableScaleDecimal is not bytes, got {:?}", avro_value - ))), - }?; + ), + }; let negative = value.sign() == Sign::Minus; - let (lo, mid, hi) = extract_decimal(value.to_bytes_be().1); + let (lo, mid, hi) = extract_decimal(value.to_bytes_be().1)?; let decimal = rust_decimal::Decimal::from_parts(lo, mid, hi, negative, scale as u32); ScalarImpl::Decimal(risingwave_common::types::Decimal::Normalized(decimal)) @@ -181,19 +181,27 @@ impl<'a> AvroParseOptions<'a> { } (Some(DataType::Varchar) | None, Value::String(s)) => s.clone().into_boxed_str().into(), // ---- Timestamp ----- - (Some(DataType::Timestamp) | None, Value::TimestampMillis(ms)) => { - i64_to_timestamp(*ms).map_err(|_| create_error())?.into() + (Some(DataType::Timestamp) | None, Value::LocalTimestampMillis(ms)) => { + Timestamp::with_millis(*ms) + .map_err(|_| create_error())? + .into() } - (Some(DataType::Timestamp) | None, Value::TimestampMicros(us)) => { - i64_to_timestamp(*us).map_err(|_| create_error())?.into() + (Some(DataType::Timestamp) | None, Value::LocalTimestampMicros(us)) => { + Timestamp::with_micros(*us) + .map_err(|_| create_error())? + .into() } // ---- TimestampTz ----- - (Some(DataType::Timestamptz), Value::TimestampMillis(ms)) => { - i64_to_timestamptz(*ms).map_err(|_| create_error())?.into() + (Some(DataType::Timestamptz) | None, Value::TimestampMillis(ms)) => { + Timestamptz::from_millis(*ms) + .ok_or_else(|| { + uncategorized!("timestamptz with milliseconds {ms} * 1000 is out of range") + })? + .into() } - (Some(DataType::Timestamptz), Value::TimestampMicros(us)) => { - i64_to_timestamptz(*us).map_err(|_| create_error())?.into() + (Some(DataType::Timestamptz) | None, Value::TimestampMicros(us)) => { + Timestamptz::from_micros(*us).into() } // ---- Interval ----- @@ -327,11 +335,11 @@ pub(crate) fn avro_decimal_to_rust_decimal( avro_decimal: AvroDecimal, _precision: usize, scale: usize, -) -> RwResult { +) -> AccessResult { let negative = !avro_decimal.is_positive(); let bytes = avro_decimal.to_vec_unsigned(); - let (lo, mid, hi) = extract_decimal(bytes); + let (lo, mid, hi) = extract_decimal(bytes)?; Ok(rust_decimal::Decimal::from_parts( lo, mid, @@ -341,34 +349,44 @@ pub(crate) fn avro_decimal_to_rust_decimal( )) } -pub(crate) fn extract_decimal(bytes: Vec) -> (u32, u32, u32) { +pub(crate) fn extract_decimal(bytes: Vec) -> AccessResult<(u32, u32, u32)> { match bytes.len() { len @ 0..=4 => { let mut pad = vec![0; 4 - len]; pad.extend_from_slice(&bytes); let lo = u32::from_be_bytes(pad.try_into().unwrap()); - (lo, 0, 0) + Ok((lo, 0, 0)) } len @ 5..=8 => { - let mid = u32::from_be_bytes(bytes[..4].to_owned().try_into().unwrap()); - let mut pad = vec![0; 8 - len]; - pad.extend_from_slice(&bytes[4..]); - let lo = u32::from_be_bytes(pad.try_into().unwrap()); - (lo, mid, 0) + let zero_len = 8 - len; + let mid_end = 4 - zero_len; + + let mut pad = vec![0; zero_len]; + pad.extend_from_slice(&bytes[..mid_end]); + let mid = u32::from_be_bytes(pad.try_into().unwrap()); + + let lo = u32::from_be_bytes(bytes[mid_end..].to_owned().try_into().unwrap()); + Ok((lo, mid, 0)) } len @ 9..=12 => { - let hi = u32::from_be_bytes(bytes[..4].to_owned().try_into().unwrap()); - let mid = u32::from_be_bytes(bytes[4..8].to_owned().try_into().unwrap()); - let mut pad = vec![0; 12 - len]; - pad.extend_from_slice(&bytes[8..]); - let lo = u32::from_be_bytes(pad.try_into().unwrap()); - (lo, mid, hi) + let zero_len = 12 - len; + let hi_end = 4 - zero_len; + let mid_end = hi_end + 4; + + let mut pad = vec![0; zero_len]; + pad.extend_from_slice(&bytes[..hi_end]); + let hi = u32::from_be_bytes(pad.try_into().unwrap()); + + let mid = u32::from_be_bytes(bytes[hi_end..mid_end].to_owned().try_into().unwrap()); + + let lo = u32::from_be_bytes(bytes[mid_end..].to_owned().try_into().unwrap()); + Ok((lo, mid, hi)) } - _ => unreachable!(), + _ => bail_uncategorized!("invalid decimal bytes length {}", bytes.len()), } } -pub fn avro_schema_skip_union(schema: &Schema) -> anyhow::Result<&Schema> { +pub fn avro_schema_skip_union(schema: &Schema) -> ConnectorResult<&Schema> { match schema { Schema::Union(union_schema) => { let inner_schema = union_schema @@ -387,7 +405,7 @@ pub fn avro_schema_skip_union(schema: &Schema) -> anyhow::Result<&Schema> { pub fn avro_extract_field_schema<'a>( schema: &'a Schema, name: Option<&'a str>, -) -> anyhow::Result<&'a Schema> { +) -> ConnectorResult<&'a Schema> { match schema { Schema::Record(RecordSchema { fields, lookup, .. }) => { let name = @@ -402,7 +420,7 @@ pub fn avro_extract_field_schema<'a>( } Schema::Array(schema) => Ok(schema), Schema::Union(_) => avro_schema_skip_union(schema), - _ => Err(anyhow::format_err!("avro schema is not a record or array")), + _ => bail!("avro schema is not a record or array"), } } @@ -413,8 +431,7 @@ pub(crate) fn unix_epoch_days() -> i32 { #[cfg(test)] mod tests { use apache_avro::Decimal as AvroDecimal; - use risingwave_common::error::{ErrorCode, RwError}; - use risingwave_common::types::{Decimal, Timestamp}; + use risingwave_common::types::{Decimal, Timestamptz}; use super::*; @@ -431,6 +448,25 @@ mod tests { let avro_decimal = AvroDecimal::from(v); let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 1).unwrap(); assert_eq!(rust_decimal, rust_decimal::Decimal::try_from(28.1).unwrap()); + + // 1.1234567891 + let value = BigInt::from(11234567891_i64); + let negative = value.sign() == Sign::Minus; + let (lo, mid, hi) = extract_decimal(value.to_bytes_be().1).unwrap(); + let decimal = rust_decimal::Decimal::from_parts(lo, mid, hi, negative, 10); + assert_eq!( + decimal, + rust_decimal::Decimal::try_from(1.1234567891).unwrap() + ); + + // 1.123456789123456789123456789 + let v = vec![3, 161, 77, 58, 146, 180, 49, 220, 100, 4, 95, 21]; + let avro_decimal = AvroDecimal::from(v); + let rust_decimal = avro_decimal_to_rust_decimal(avro_decimal, 28, 27).unwrap(); + assert_eq!( + rust_decimal, + rust_decimal::Decimal::from_str("1.123456789123456789123456789").unwrap() + ); } /// Convert Avro value to datum.For now, support the following [Avro type](https://avro.apache.org/docs/current/spec.html). @@ -447,34 +483,34 @@ mod tests { value: Value, value_schema: &Schema, shape: &DataType, - ) -> RwResult { + ) -> crate::error::ConnectorResult { AvroParseOptions { schema: Some(value_schema), relax_numeric: true, } .parse(&value, Some(shape)) - .map_err(|err| RwError::from(ErrorCode::InternalError(format!("{:?}", err)))) + .map_err(Into::into) } #[test] - fn test_avro_timestamp_micros() { - let v1 = Value::TimestampMicros(1620000000000); - let v2 = Value::TimestampMillis(1620000000); + fn test_avro_timestamptz_micros() { + let v1 = Value::TimestampMicros(1620000000000000); + let v2 = Value::TimestampMillis(1620000000000); let value_schema1 = Schema::TimestampMicros; let value_schema2 = Schema::TimestampMillis; - let datum1 = from_avro_value(v1, &value_schema1, &DataType::Timestamp).unwrap(); - let datum2 = from_avro_value(v2, &value_schema2, &DataType::Timestamp).unwrap(); + let datum1 = from_avro_value(v1, &value_schema1, &DataType::Timestamptz).unwrap(); + let datum2 = from_avro_value(v2, &value_schema2, &DataType::Timestamptz).unwrap(); assert_eq!( datum1, - Some(ScalarImpl::Timestamp(Timestamp::new( - "2021-05-03T00:00:00".parse().unwrap() - ))) + Some(ScalarImpl::Timestamptz( + Timestamptz::from_str("2021-05-03T00:00:00Z").unwrap() + )) ); assert_eq!( datum2, - Some(ScalarImpl::Timestamp(Timestamp::new( - "2021-05-03T00:00:00".parse().unwrap() - ))) + Some(ScalarImpl::Timestamptz( + Timestamptz::from_str("2021-05-03T00:00:00Z").unwrap() + )) ); } @@ -498,7 +534,7 @@ mod tests { assert_eq!( resp, Some(ScalarImpl::Decimal(Decimal::Normalized( - rust_decimal::Decimal::from_str("4.557430887741865791").unwrap() + rust_decimal::Decimal::from_str("0.017802464409370431").unwrap() ))) ); } diff --git a/src/connector/src/parser/unified/debezium.rs b/src/connector/src/parser/unified/debezium.rs index 7291b1b359735..589fdfc8cfc45 100644 --- a/src/connector/src/parser/unified/debezium.rs +++ b/src/connector/src/parser/unified/debezium.rs @@ -12,16 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use risingwave_common::types::{DataType, Datum, ScalarImpl}; -use super::{Access, AccessError, ChangeEvent, ChangeEventOperation}; +use super::{Access, AccessError, AccessResult, ChangeEvent, ChangeEventOperation}; use crate::parser::TransactionControl; use crate::source::{ConnectorProperties, SourceColumnDesc}; pub struct DebeziumChangeEvent { value_accessor: Option, key_accessor: Option, + is_mongodb: bool, } const BEFORE: &str = "before"; @@ -41,7 +41,7 @@ pub const DEBEZIUM_TRANSACTION_STATUS_COMMIT: &str = "END"; pub fn parse_transaction_meta( accessor: &impl Access, connector_props: &ConnectorProperties, -) -> std::result::Result { +) -> AccessResult { if let (Some(ScalarImpl::Utf8(status)), Some(ScalarImpl::Utf8(id))) = ( accessor.access(&[TRANSACTION_STATUS], Some(&DataType::Varchar))?, accessor.access(&[TRANSACTION_ID], Some(&DataType::Varchar))?, @@ -94,6 +94,16 @@ where Self { value_accessor, key_accessor, + is_mongodb: false, + } + } + + pub fn new_mongodb_event(key_accessor: Option, value_accessor: Option) -> Self { + assert!(key_accessor.is_some() || value_accessor.is_some()); + Self { + value_accessor, + key_accessor, + is_mongodb: true, } } @@ -103,13 +113,12 @@ where pub(crate) fn transaction_control( &self, connector_props: &ConnectorProperties, - ) -> Result { - let Some(accessor) = &self.value_accessor else { - return Err(AccessError::Other(anyhow!( - "value_accessor must be provided to parse transaction metadata" - ))); - }; - parse_transaction_meta(accessor, connector_props) + ) -> Option { + // Ignore if `value_accessor` is not provided or there's any error when + // trying to parse the transaction metadata. + self.value_accessor + .as_ref() + .and_then(|accessor| parse_transaction_meta(accessor, connector_props).ok()) } } @@ -120,6 +129,16 @@ where fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult { match self.op()? { ChangeEventOperation::Delete => { + // For delete events of MongoDB, the "before" and "after" field both are null in the value, + // we need to extract the _id field from the key. + if self.is_mongodb && desc.name == "_id" { + return self + .key_accessor + .as_ref() + .expect("key_accessor must be provided for delete operation") + .access(&[&desc.name], Some(&desc.data_type)); + } + if let Some(va) = self.value_accessor.as_ref() { va.access(&[BEFORE, &desc.name], Some(&desc.data_type)) } else { @@ -160,14 +179,31 @@ where } } -pub struct MongoProjection { +pub struct MongoJsonAccess { accessor: A, } -pub fn extract_bson_id(id_type: &DataType, bson_doc: &serde_json::Value) -> anyhow::Result { - let id_field = bson_doc - .get("_id") - .ok_or_else(|| anyhow::format_err!("Debezuim Mongo requires document has a `_id` field"))?; +pub fn extract_bson_id(id_type: &DataType, bson_doc: &serde_json::Value) -> AccessResult { + let id_field = if let Some(value) = bson_doc.get("_id") { + value + } else { + bson_doc + }; + + let type_error = || AccessError::TypeError { + expected: id_type.to_string(), + got: match id_field { + serde_json::Value::Null => "null", + serde_json::Value::Bool(_) => "bool", + serde_json::Value::Number(_) => "number", + serde_json::Value::String(_) => "string", + serde_json::Value::Array(_) => "array", + serde_json::Value::Object(_) => "object", + } + .to_owned(), + value: id_field.to_string(), + }; + let id: Datum = match id_type { DataType::Jsonb => ScalarImpl::Jsonb(id_field.clone().into()).into(), DataType::Varchar => match id_field { @@ -175,7 +211,7 @@ pub fn extract_bson_id(id_type: &DataType, bson_doc: &serde_json::Value) -> anyh serde_json::Value::Object(obj) if obj.contains_key("$oid") => Some(ScalarImpl::Utf8( obj["$oid"].as_str().to_owned().unwrap_or_default().into(), )), - _ => anyhow::bail!("Can not convert bson {:?} to {:?}", id_field, id_type), + _ => return Err(type_error()), }, DataType::Int32 => { if let serde_json::Value::Object(ref obj) = id_field @@ -184,7 +220,7 @@ pub fn extract_bson_id(id_type: &DataType, bson_doc: &serde_json::Value) -> anyh let int_str = obj["$numberInt"].as_str().unwrap_or_default(); Some(ScalarImpl::Int32(int_str.parse().unwrap_or_default())) } else { - anyhow::bail!("Can not convert bson {:?} to {:?}", id_field, id_type) + return Err(type_error()); } } DataType::Int64 => { @@ -194,20 +230,20 @@ pub fn extract_bson_id(id_type: &DataType, bson_doc: &serde_json::Value) -> anyh let int_str = obj["$numberLong"].as_str().unwrap_or_default(); Some(ScalarImpl::Int64(int_str.parse().unwrap_or_default())) } else { - anyhow::bail!("Can not convert bson {:?} to {:?}", id_field, id_type) + return Err(type_error()); } } _ => unreachable!("DebeziumMongoJsonParser::new must ensure _id column datatypes."), }; Ok(id) } -impl MongoProjection { +impl MongoJsonAccess { pub fn new(accessor: A) -> Self { Self { accessor } } } -impl Access for MongoProjection +impl Access for MongoJsonAccess where A: Access, { @@ -221,10 +257,37 @@ where &bson_doc.take(), )?) } else { - unreachable!("the result of access must match the type_expected") + // fail to extract the "_id" field from the message payload + Err(AccessError::Undefined { + name: "_id".to_string(), + path: path[0].to_string(), + })? } } ["after" | "before", "payload"] => self.access(&[path[0]], Some(&DataType::Jsonb)), + // To handle a DELETE message, we need to extract the "_id" field from the message key, because it is not in the payload. + // In addition, the "_id" field is named as "id" in the key. An example of message key: + // {"schema":null,"payload":{"id":"{\"$oid\": \"65bc9fb6c485f419a7a877fe\"}"}} + ["_id"] => { + let ret = self.accessor.access(path, type_expected); + if matches!(ret, Err(AccessError::Undefined { .. })) { + let id_bson = self.accessor.access(&["id"], Some(&DataType::Jsonb))?; + if let Some(ScalarImpl::Jsonb(bson_doc)) = id_bson { + Ok(extract_bson_id( + type_expected.unwrap_or(&DataType::Jsonb), + &bson_doc.take(), + )?) + } else { + // fail to extract the "_id" field from the message key + Err(AccessError::Undefined { + name: "_id".to_string(), + path: "id".to_string(), + })? + } + } else { + ret + } + } _ => self.accessor.access(path, type_expected), } } diff --git a/src/connector/src/parser/unified/json.rs b/src/connector/src/parser/unified/json.rs index ecda30430e12b..a765f333ef314 100644 --- a/src/connector/src/parser/unified/json.rs +++ b/src/connector/src/parser/unified/json.rs @@ -29,6 +29,7 @@ use simd_json::prelude::{ TypedValue, ValueAsContainer, ValueAsScalar, ValueObjectAccess, ValueTryAsScalar, }; use simd_json::{BorrowedValue, ValueType}; +use thiserror_ext::AsReport; use super::{Access, AccessError, AccessResult}; use crate::parser::common::json_object_get_case_insensitive; @@ -358,7 +359,7 @@ impl JsonParseOptions { .as_bytes(); let decimal = BigInt::from_signed_bytes_be(value); let negative = decimal.sign() == Sign::Minus; - let (lo, mid, hi) = extract_decimal(decimal.to_bytes_be().1); + let (lo, mid, hi) = extract_decimal(decimal.to_bytes_be().1)?; let decimal = rust_decimal::Decimal::from_parts(lo, mid, hi, negative, scale as u32); ScalarImpl::Decimal(Decimal::Normalized(decimal)) @@ -468,7 +469,7 @@ impl JsonParseOptions { // TODO: is it possible to unify the logging with the one in `do_action`? static LOG_SUPPERSSER: LazyLock = LazyLock::new(LogSuppresser::default); if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { - tracing::warn!(%error, suppressed_count, "undefined nested field, padding with `NULL`"); + tracing::warn!(error = %error.as_report(), suppressed_count, "undefined nested field, padding with `NULL`"); } &BorrowedValue::Static(simd_json::StaticNode::Null) }); diff --git a/src/connector/src/parser/unified/mod.rs b/src/connector/src/parser/unified/mod.rs index bd52f4628ef2f..31667cd42e47d 100644 --- a/src/connector/src/parser/unified/mod.rs +++ b/src/connector/src/parser/unified/mod.rs @@ -17,11 +17,13 @@ use auto_impl::auto_impl; use risingwave_common::types::{DataType, Datum}; use thiserror::Error; +use thiserror_ext::Macro; use self::avro::AvroAccess; use self::bytes::BytesAccess; use self::json::JsonAccess; use self::protobuf::ProtobufAccess; +use crate::parser::unified::debezium::MongoJsonAccess; use crate::source::SourceColumnDesc; pub mod avro; @@ -45,6 +47,7 @@ pub enum AccessImpl<'a, 'b> { Bytes(BytesAccess<'a>), Protobuf(ProtobufAccess), Json(JsonAccess<'a, 'b>), + MongoJson(MongoJsonAccess>), } impl Access for AccessImpl<'_, '_> { @@ -54,6 +57,7 @@ impl Access for AccessImpl<'_, '_> { Self::Bytes(accessor) => accessor.access(path, type_expected), Self::Protobuf(accessor) => accessor.access(path, type_expected), Self::Json(accessor) => accessor.access(path, type_expected), + Self::MongoJson(accessor) => accessor.access(path, type_expected), } } } @@ -86,7 +90,8 @@ where } } -#[derive(Error, Debug)] +#[derive(Error, Debug, Macro)] +#[thiserror_ext(macro(mangle))] pub enum AccessError { #[error("Undefined field `{name}` at `{path}`")] Undefined { name: String, path: String }, @@ -98,10 +103,8 @@ pub enum AccessError { }, #[error("Unsupported data type `{ty}`")] UnsupportedType { ty: String }, - #[error(transparent)] - Other( - #[from] - #[backtrace] - anyhow::Error, - ), + + /// Errors that are not categorized into variants above. + #[error("{message}")] + Uncategorized { message: String }, } diff --git a/src/connector/src/parser/unified/protobuf.rs b/src/connector/src/parser/unified/protobuf.rs index 3fdef073faa23..cd9178c7dd08d 100644 --- a/src/connector/src/parser/unified/protobuf.rs +++ b/src/connector/src/parser/unified/protobuf.rs @@ -14,16 +14,14 @@ use std::sync::{Arc, LazyLock}; -use anyhow::anyhow; use prost_reflect::{DescriptorPool, DynamicMessage, ReflectMessage}; -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::RwError; use risingwave_common::log::LogSuppresser; use risingwave_common::types::DataType; +use thiserror_ext::AsReport; use super::{Access, AccessResult}; use crate::parser::from_protobuf_value; -use crate::parser::unified::AccessError; +use crate::parser::unified::{uncategorized, AccessError}; pub struct ProtobufAccess { message: DynamicMessage, @@ -46,18 +44,16 @@ impl Access for ProtobufAccess { .message .descriptor() .get_field_by_name(path[0]) - .ok_or_else(|| { - let err_msg = format!("protobuf schema don't have field {}", path[0]); + .ok_or_else(|| uncategorized!("protobuf schema don't have field {}", path[0])) + .inspect_err(|e| { static LOG_SUPPERSSER: LazyLock = LazyLock::new(LogSuppresser::default); if let Ok(suppressed_count) = LOG_SUPPERSSER.check() { - tracing::error!(suppressed_count, err_msg); + tracing::error!(suppressed_count, "{}", e.as_report()); } - RwError::from(ProtocolError(err_msg)) - }) - .map_err(|e| AccessError::Other(anyhow!(e)))?; + })?; let value = self.message.get_field(&field_desc); + from_protobuf_value(&field_desc, &value, &self.descriptor_pool) - .map_err(|e| AccessError::Other(anyhow!(e))) } } diff --git a/src/connector/src/parser/unified/upsert.rs b/src/connector/src/parser/unified/upsert.rs index 648a667e2b2d8..9129f0d16d864 100644 --- a/src/connector/src/parser/unified/upsert.rs +++ b/src/connector/src/parser/unified/upsert.rs @@ -13,7 +13,7 @@ // limitations under the License. use risingwave_common::types::DataType; -use risingwave_pb::plan_common::AdditionalColumnType; +use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; use super::{Access, ChangeEvent, ChangeEventOperation}; use crate::parser::unified::AccessError; @@ -105,8 +105,8 @@ where } fn access_field(&self, desc: &SourceColumnDesc) -> super::AccessResult { - match desc.additional_column_type { - AdditionalColumnType::Key => { + match desc.additional_column.column_type { + Some(AdditionalColumnType::Key(_)) => { if let Some(key_as_column_name) = &self.key_column_name && &desc.name == key_as_column_name { @@ -115,9 +115,7 @@ where self.access(&["key", &desc.name], Some(&desc.data_type)) } } - AdditionalColumnType::Unspecified | AdditionalColumnType::Normal => { - self.access(&["value", &desc.name], Some(&desc.data_type)) - } + None => self.access(&["value", &desc.name], Some(&desc.data_type)), _ => unreachable!(), } } diff --git a/src/connector/src/parser/unified/util.rs b/src/connector/src/parser/unified/util.rs index b7b00667e7de4..948190edf685a 100644 --- a/src/connector/src/parser/unified/util.rs +++ b/src/connector/src/parser/unified/util.rs @@ -12,9 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::{ErrorCode, RwError}; - -use super::{Access, AccessError, AccessResult, ChangeEvent}; +use super::{Access, AccessResult, ChangeEvent}; use crate::parser::unified::ChangeEventOperation; use crate::parser::SourceStreamChunkRowWriter; use crate::source::SourceColumnDesc; @@ -45,9 +43,3 @@ pub fn apply_row_accessor_on_stream_chunk_writer( ) -> AccessResult<()> { writer.insert(|column| accessor.access(&[&column.name], Some(&column.data_type))) } - -impl From for RwError { - fn from(val: AccessError) -> Self { - ErrorCode::InternalError(format!("AccessError: {:?}", val)).into() - } -} diff --git a/src/connector/src/parser/upsert_parser.rs b/src/connector/src/parser/upsert_parser.rs index 99a2c19c33e78..048fd0beca3ff 100644 --- a/src/connector/src/parser/upsert_parser.rs +++ b/src/connector/src/parser/upsert_parser.rs @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; -use risingwave_pb::plan_common::AdditionalColumnType; +use risingwave_common::bail; +use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; use super::bytes_parser::BytesAccessBuilder; use super::unified::upsert::UpsertChangeEvent; @@ -24,6 +23,7 @@ use super::{ AccessBuilderImpl, ByteStreamSourceParser, BytesProperties, EncodingProperties, EncodingType, SourceStreamChunkRowWriter, SpecificParserConfig, }; +use crate::error::ConnectorResult; use crate::parser::ParserFormat; use crate::source::{SourceColumnDesc, SourceContext, SourceContextRef}; @@ -38,22 +38,23 @@ pub struct UpsertParser { async fn build_accessor_builder( config: EncodingProperties, encoding_type: EncodingType, -) -> Result { +) -> ConnectorResult { match config { EncodingProperties::Json(_) | EncodingProperties::Protobuf(_) | EncodingProperties::Avro(_) => { Ok(AccessBuilderImpl::new_default(config, encoding_type).await?) } - _ => Err(RwError::from(ProtocolError( - "unsupported encoding for Upsert".to_string(), - ))), + _ => bail!("unsupported encoding for Upsert"), } } pub fn get_key_column_name(columns: &[SourceColumnDesc]) -> Option { columns.iter().find_map(|column| { - if column.additional_column_type == AdditionalColumnType::Key { + if matches!( + column.additional_column.column_type, + Some(AdditionalColumnType::Key(_)) + ) { Some(column.name.clone()) } else { None @@ -66,7 +67,7 @@ impl UpsertParser { props: SpecificParserConfig, rw_columns: Vec, source_ctx: SourceContextRef, - ) -> Result { + ) -> ConnectorResult { // check whether columns has Key as AdditionalColumnType, if so, the key accessor should be // bytes let key_builder = if let Some(key_column_name) = get_key_column_name(&rw_columns) { @@ -95,7 +96,7 @@ impl UpsertParser { key: Option>, payload: Option>, mut writer: SourceStreamChunkRowWriter<'_>, - ) -> Result<()> { + ) -> ConnectorResult<()> { let mut row_op: UpsertChangeEvent, AccessImpl<'_, '_>> = UpsertChangeEvent::default(); let mut change_event_op = ChangeEventOperation::Delete; @@ -133,7 +134,7 @@ impl ByteStreamSourceParser for UpsertParser { key: Option>, payload: Option>, writer: SourceStreamChunkRowWriter<'a>, - ) -> Result<()> { + ) -> ConnectorResult<()> { self.parse_inner(key, payload, writer).await } } diff --git a/src/connector/src/parser/util.rs b/src/connector/src/parser/util.rs index 4e0bd6f2f3f0f..eeaa09bcc031d 100644 --- a/src/connector/src/parser/util.rs +++ b/src/connector/src/parser/util.rs @@ -13,18 +13,20 @@ // limitations under the License. use std::collections::HashMap; +use anyhow::Context; use bytes::Bytes; use reqwest::Url; -use risingwave_common::error::ErrorCode::{InvalidParameterValue, ProtocolError}; -use risingwave_common::error::{Result, RwError}; +use risingwave_common::bail; use risingwave_common::types::Datum; +use risingwave_pb::data::DataType as PbDataType; use crate::aws_utils::load_file_descriptor_from_s3; use crate::common::AwsAuthProps; +use crate::error::ConnectorResult; use crate::source::SourceMeta; /// get kafka topic name -pub(super) fn get_kafka_topic(props: &HashMap) -> Result<&String> { +pub(super) fn get_kafka_topic(props: &HashMap) -> ConnectorResult<&String> { const KAFKA_TOPIC_KEY1: &str = "kafka.topic"; const KAFKA_TOPIC_KEY2: &str = "topic"; @@ -35,30 +37,28 @@ pub(super) fn get_kafka_topic(props: &HashMap) -> Result<&String return Ok(topic); } - Err(RwError::from(ProtocolError(format!( + // config + bail!( "Must specify '{}' or '{}'", - KAFKA_TOPIC_KEY1, KAFKA_TOPIC_KEY2, - )))) + KAFKA_TOPIC_KEY1, + KAFKA_TOPIC_KEY2 + ) } /// download bytes from http(s) url -pub(super) async fn download_from_http(location: &Url) -> Result { - let res = reqwest::get(location.clone()).await.map_err(|e| { - InvalidParameterValue(format!( - "failed to make request to URL: {}, err: {}", - location, e - )) - })?; - if !res.status().is_success() { - return Err(RwError::from(InvalidParameterValue(format!( - "Http request err, URL: {}, status code: {}", - location, - res.status() - )))); - } - res.bytes() +pub(super) async fn download_from_http(location: &Url) -> ConnectorResult { + let res = reqwest::get(location.clone()) + .await + .with_context(|| format!("failed to make request to {location}"))? + .error_for_status() + .with_context(|| format!("http request failed for {location}"))?; + + let bytes = res + .bytes() .await - .map_err(|e| InvalidParameterValue(format!("failed to read HTTP body: {}", e)).into()) + .with_context(|| format!("failed to read HTTP body of {location}"))?; + + Ok(bytes) } // For parser that doesn't support key currently @@ -68,9 +68,7 @@ macro_rules! only_parse_payload { if let Some(payload) = $payload { $self.parse_inner(payload, $writer).await } else { - Err(RwError::from(ErrorCode::InternalError( - "Empty payload with nonempty key".into(), - ))) + risingwave_common::bail!("empty payload with non-empty key") } }; } @@ -95,20 +93,23 @@ macro_rules! extract_key_config { /// * local file, for on-premise or testing. /// * http/https, for common usage. /// * s3 file location format: -pub(super) async fn bytes_from_url(url: &Url, config: Option<&AwsAuthProps>) -> Result> { +pub(super) async fn bytes_from_url( + url: &Url, + config: Option<&AwsAuthProps>, +) -> ConnectorResult> { match (url.scheme(), config) { // TODO(Tao): support local file only when it's compiled in debug mode. ("file", _) => { let path = url .to_file_path() - .map_err(|()| InvalidParameterValue(format!("illegal path: {url}")))?; - Ok(std::fs::read(path)?) + .ok() + .with_context(|| format!("illegal path: {url}"))?; + Ok(std::fs::read(&path) + .with_context(|| format!("failed to read file from `{}`", path.display()))?) } ("https" | "http", _) => Ok(download_from_http(url).await?.into()), ("s3", Some(config)) => load_file_descriptor_from_s3(url, config).await, - (scheme, _) => Err(RwError::from(InvalidParameterValue(format!( - "path scheme {scheme} is not supported", - )))), + (scheme, _) => bail!("path scheme `{scheme}` is not supported"), } } @@ -121,7 +122,18 @@ pub fn extreact_timestamp_from_meta(meta: &SourceMeta) -> Option { pub fn extract_headers_from_meta(meta: &SourceMeta) -> Option { match meta { - SourceMeta::Kafka(kafka_meta) => kafka_meta.extract_headers(), + SourceMeta::Kafka(kafka_meta) => kafka_meta.extract_headers(), /* expect output of type `array[struct]` */ + _ => None, + } +} + +pub fn extract_header_inner_from_meta( + meta: &SourceMeta, + inner_field: &str, + data_type: Option<&PbDataType>, +) -> Option { + match meta { + SourceMeta::Kafka(kafka_meta) => kafka_meta.extract_header_inner(inner_field, data_type), /* expect output of type `bytea` or `varchar` */ _ => None, } } diff --git a/src/connector/src/schema/avro.rs b/src/connector/src/schema/avro.rs index 2fe741a27e24e..553f6e71efca9 100644 --- a/src/connector/src/schema/avro.rs +++ b/src/connector/src/schema/avro.rs @@ -23,8 +23,8 @@ use super::schema_registry::{ SchemaRegistryAuth, }; use super::{ - SchemaFetchError, KEY_MESSAGE_NAME_KEY, MESSAGE_NAME_KEY, NAME_STRATEGY_KEY, - SCHEMA_REGISTRY_KEY, + invalid_option_error, InvalidOptionError, SchemaFetchError, KEY_MESSAGE_NAME_KEY, + MESSAGE_NAME_KEY, NAME_STRATEGY_KEY, SCHEMA_REGISTRY_KEY, }; pub struct SchemaWithId { @@ -36,8 +36,8 @@ impl TryFrom for SchemaWithId { type Error = SchemaFetchError; fn try_from(fetched: ConfluentSchema) -> Result { - let parsed = - AvroSchema::parse_str(&fetched.content).map_err(|e| SchemaFetchError(e.to_string()))?; + let parsed = AvroSchema::parse_str(&fetched.content) + .map_err(|e| SchemaFetchError::SchemaCompile(e.into()))?; Ok(Self { schema: Arc::new(parsed), id: fetched.id, @@ -52,14 +52,14 @@ pub async fn fetch_schema( ) -> Result<(SchemaWithId, SchemaWithId), SchemaFetchError> { let schema_location = format_options .get(SCHEMA_REGISTRY_KEY) - .ok_or_else(|| SchemaFetchError(format!("{SCHEMA_REGISTRY_KEY} required")))? + .ok_or_else(|| invalid_option_error!("{SCHEMA_REGISTRY_KEY} required"))? .clone(); let client_config = format_options.into(); let name_strategy = format_options .get(NAME_STRATEGY_KEY) .map(|s| { name_strategy_from_str(s) - .ok_or_else(|| SchemaFetchError(format!("unrecognized strategy {s}"))) + .ok_or_else(|| invalid_option_error!("unrecognized strategy {s}")) }) .transpose()? .unwrap_or_default(); @@ -78,8 +78,7 @@ pub async fn fetch_schema( key_record_name, val_record_name, ) - .await - .map_err(|e| SchemaFetchError(e.to_string()))?; + .await?; Ok((key_schema.try_into()?, val_schema.try_into()?)) } @@ -91,7 +90,7 @@ async fn fetch_schema_inner( topic: &str, key_record_name: Option<&str>, val_record_name: Option<&str>, -) -> Result<(ConfluentSchema, ConfluentSchema), risingwave_common::error::RwError> { +) -> Result<(ConfluentSchema, ConfluentSchema), SchemaFetchError> { let urls = handle_sr_list(schema_location)?; let client = Client::new(urls, client_config)?; diff --git a/src/connector/src/schema/mod.rs b/src/connector/src/schema/mod.rs index d2dcbb9d29bf3..8d2a9ae780572 100644 --- a/src/connector/src/schema/mod.rs +++ b/src/connector/src/schema/mod.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::error::ConnectorError; + pub mod avro; pub mod protobuf; pub mod schema_registry; @@ -22,5 +24,30 @@ const SCHEMA_LOCATION_KEY: &str = "schema.location"; const SCHEMA_REGISTRY_KEY: &str = "schema.registry"; const NAME_STRATEGY_KEY: &str = "schema.registry.name.strategy"; -#[derive(Debug)] -pub struct SchemaFetchError(pub String); +#[derive(Debug, thiserror::Error, thiserror_ext::Macro)] +#[error("Invalid option: {message}")] +pub struct InvalidOptionError { + message: String, + // #[backtrace] + // source: Option, +} + +#[derive(Debug, thiserror::Error)] +pub enum SchemaFetchError { + #[error(transparent)] + InvalidOption(#[from] InvalidOptionError), + #[error(transparent)] + Request(#[from] schema_registry::ConcurrentRequestError), + #[error("schema compilation error: {0}")] + SchemaCompile( + #[source] + #[backtrace] + risingwave_common::error::BoxedError, + ), + #[error("{0}")] // source+{0} is effectively transparent but allows backtrace + YetToMigrate( + #[source] + #[backtrace] + ConnectorError, + ), +} diff --git a/src/connector/src/schema/protobuf.rs b/src/connector/src/schema/protobuf.rs index 096a8ed95de7f..0d6116121977d 100644 --- a/src/connector/src/schema/protobuf.rs +++ b/src/connector/src/schema/protobuf.rs @@ -16,7 +16,10 @@ use std::collections::BTreeMap; use prost_reflect::MessageDescriptor; -use super::{SchemaFetchError, MESSAGE_NAME_KEY, SCHEMA_LOCATION_KEY}; +use super::{ + invalid_option_error, InvalidOptionError, SchemaFetchError, MESSAGE_NAME_KEY, + SCHEMA_LOCATION_KEY, +}; use crate::common::AwsAuthProps; use crate::parser::{EncodingProperties, ProtobufParserConfig, ProtobufProperties}; @@ -27,15 +30,15 @@ pub async fn fetch_descriptor( ) -> Result { let row_schema_location = format_options .get(SCHEMA_LOCATION_KEY) - .ok_or_else(|| SchemaFetchError(format!("{SCHEMA_LOCATION_KEY} required")))? + .ok_or_else(|| invalid_option_error!("{SCHEMA_LOCATION_KEY} required"))? .clone(); let message_name = format_options .get(MESSAGE_NAME_KEY) - .ok_or_else(|| SchemaFetchError(format!("{MESSAGE_NAME_KEY} required")))? + .ok_or_else(|| invalid_option_error!("{MESSAGE_NAME_KEY} required"))? .clone(); if row_schema_location.starts_with("s3") && aws_auth_props.is_none() { - return Err(SchemaFetchError("s3 URL not supported yet".into())); + return Err(invalid_option_error!("s3 URL not supported yet").into()); } let enc = EncodingProperties::Protobuf(ProtobufProperties { @@ -52,6 +55,6 @@ pub async fn fetch_descriptor( // This reversed dependency will be fixed when we support schema registry. let conf = ProtobufParserConfig::new(enc) .await - .map_err(|e| SchemaFetchError(e.to_string()))?; + .map_err(SchemaFetchError::YetToMigrate)?; Ok(conf.message_descriptor) } diff --git a/src/connector/src/schema/schema_registry/client.rs b/src/connector/src/schema/schema_registry/client.rs index 0206524c0f70c..21b0d4a7f6586 100644 --- a/src/connector/src/schema/schema_registry/client.rs +++ b/src/connector/src/schema/schema_registry/client.rs @@ -19,11 +19,11 @@ use std::sync::Arc; use futures::future::select_all; use itertools::Itertools; use reqwest::{Method, Url}; -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; use serde::de::DeserializeOwned; +use thiserror_ext::AsReport as _; use super::util::*; +use crate::schema::{invalid_option_error, InvalidOptionError}; pub const SCHEMA_REGISTRY_USERNAME: &str = "schema.registry.username"; pub const SCHEMA_REGISTRY_PASSWORD: &str = "schema.registry.password"; @@ -61,8 +61,20 @@ pub struct Client { password: Option, } +#[derive(Debug, thiserror::Error)] +#[error("all request confluent registry all timeout, {context}\n{}", errs.iter().map(|e| format!("\t{}", e.as_report())).join("\n"))] +pub struct ConcurrentRequestError { + errs: Vec>, + context: String, +} + +type SrResult = Result; + impl Client { - pub(crate) fn new(url: Vec, client_config: &SchemaRegistryAuth) -> Result { + pub(crate) fn new( + url: Vec, + client_config: &SchemaRegistryAuth, + ) -> Result { let valid_urls = url .iter() .map(|url| (url.cannot_be_a_base(), url)) @@ -70,10 +82,7 @@ impl Client { .map(|(_, url)| url.clone()) .collect_vec(); if valid_urls.is_empty() { - return Err(RwError::from(ProtocolError(format!( - "no valid url provided, got {:?}", - url - )))); + return Err(invalid_option_error!("non-base: {}", url.iter().join(" "))); } else { tracing::debug!( "schema registry client will use url {:?} to connect", @@ -81,9 +90,8 @@ impl Client { ); } - let inner = reqwest::Client::builder().build().map_err(|e| { - RwError::from(ProtocolError(format!("build reqwest client failed {}", e))) - })?; + // `unwrap` as the builder is not affected by any input right now + let inner = reqwest::Client::builder().build().unwrap(); Ok(Client { inner, @@ -97,7 +105,7 @@ impl Client { &'a self, method: Method, path: &'a [&'a (impl AsRef + ?Sized + Debug + ToString)], - ) -> Result + ) -> SrResult where T: DeserializeOwned + Send + Sync + 'static, { @@ -124,22 +132,20 @@ impl Client { let _ = remaining.iter().map(|ele| ele.abort()); return Ok(res); } - Ok(Err(e)) => errs.push(e), - Err(e) => errs.push(RwError::from(e)), + Ok(Err(e)) => errs.push(itertools::Either::Left(e)), + Err(e) => errs.push(itertools::Either::Right(e)), } fut_req = remaining; } - Err(RwError::from(ProtocolError(format!( - "all request confluent registry all timeout, req path {:?}, urls {:?}, err: {:?}", - path, - self.url, - errs.iter().map(|e| e.to_string()).collect_vec() - )))) + Err(ConcurrentRequestError { + errs, + context: format!("req path {:?}, urls {}", path, self.url.iter().join(" ")), + }) } /// get schema by id - pub async fn get_schema_by_id(&self, id: i32) -> Result { + pub async fn get_schema_by_id(&self, id: i32) -> SrResult { let res: GetByIdResp = self .concurrent_req(Method::GET, &["schemas", "ids", &id.to_string()]) .await?; @@ -150,12 +156,12 @@ impl Client { } /// get the latest schema of the subject - pub async fn get_schema_by_subject(&self, subject: &str) -> Result { + pub async fn get_schema_by_subject(&self, subject: &str) -> SrResult { self.get_subject(subject).await.map(|s| s.schema) } /// get the latest version of the subject - pub async fn get_subject(&self, subject: &str) -> Result { + pub async fn get_subject(&self, subject: &str) -> SrResult { let res: GetBySubjectResp = self .concurrent_req(Method::GET, &["subjects", subject, "versions", "latest"]) .await?; @@ -174,7 +180,7 @@ impl Client { pub async fn get_subject_and_references( &self, subject: &str, - ) -> Result<(Subject, Vec)> { + ) -> SrResult<(Subject, Vec)> { let mut subjects = vec![]; let mut visited = HashSet::new(); let mut queue = vec![(subject.to_owned(), "latest".to_owned())]; diff --git a/src/connector/src/schema/schema_registry/mod.rs b/src/connector/src/schema/schema_registry/mod.rs index 3ae5fdf1a7371..8149c97061ac2 100644 --- a/src/connector/src/schema/schema_registry/mod.rs +++ b/src/connector/src/schema/schema_registry/mod.rs @@ -15,11 +15,11 @@ mod client; mod util; pub use client::*; -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::RwError; use risingwave_pb::catalog::SchemaRegistryNameStrategy as PbSchemaRegistryNameStrategy; pub(crate) use util::*; +use super::{invalid_option_error, InvalidOptionError}; + pub fn name_strategy_from_str(value: &str) -> Option { match value { "topic_name_strategy" => Some(PbSchemaRegistryNameStrategy::Unspecified), @@ -34,29 +34,27 @@ pub fn get_subject_by_strategy( topic: &str, record: Option<&str>, is_key: bool, -) -> Result { - let build_error_lack_field = |ns: &PbSchemaRegistryNameStrategy, expect: &str| -> RwError { - RwError::from(ProtocolError(format!( - "{} expect num-empty field {}", - ns.as_str_name(), - expect, - ))) - }; +) -> Result { let record_option_name = if is_key { "key.message" } else { "message" }; + let build_error_lack_field = || { + invalid_option_error!( + "{} expect non-empty field {}", + name_strategy.as_str_name(), + record_option_name, + ) + }; match name_strategy { PbSchemaRegistryNameStrategy::Unspecified => { // default behavior let suffix = if is_key { "key" } else { "value" }; Ok(format!("{topic}-{suffix}",)) } - ns @ PbSchemaRegistryNameStrategy::RecordNameStrategy => { - let record_name = - record.ok_or_else(|| build_error_lack_field(ns, record_option_name))?; + PbSchemaRegistryNameStrategy::RecordNameStrategy => { + let record_name = record.ok_or_else(build_error_lack_field)?; Ok(record_name.to_string()) } - ns @ PbSchemaRegistryNameStrategy::TopicRecordNameStrategy => { - let record_name = - record.ok_or_else(|| build_error_lack_field(ns, record_option_name))?; + PbSchemaRegistryNameStrategy::TopicRecordNameStrategy => { + let record_name = record.ok_or_else(build_error_lack_field)?; Ok(format!("{topic}-{record_name}")) } } diff --git a/src/connector/src/schema/schema_registry/util.rs b/src/connector/src/schema/schema_registry/util.rs index 7e621364d43bc..0d43f33baa31c 100644 --- a/src/connector/src/schema/schema_registry/util.rs +++ b/src/connector/src/schema/schema_registry/util.rs @@ -15,15 +15,14 @@ use std::fmt::Debug; use std::sync::Arc; -use byteorder::{BigEndian, ByteOrder}; use reqwest::Method; -use risingwave_common::error::ErrorCode::{InternalError, ProtocolError}; -use risingwave_common::error::{Result, RwError}; use serde::de::DeserializeOwned; use serde_derive::Deserialize; use url::{ParseError, Url}; -pub fn handle_sr_list(addr: &str) -> Result> { +use crate::schema::{bail_invalid_option_error, InvalidOptionError}; + +pub fn handle_sr_list(addr: &str) -> Result, InvalidOptionError> { let segment = addr.split(',').collect::>(); let mut errs: Vec = Vec::with_capacity(segment.len()); let mut urls = Vec::with_capacity(segment.len()); @@ -34,10 +33,7 @@ pub fn handle_sr_list(addr: &str) -> Result> { } } if urls.is_empty() { - return Err(RwError::from(ProtocolError(format!( - "no valid url provided, got {:?}", - errs - )))); + bail_invalid_option_error!("no valid url provided, errs: {errs:?}"); } tracing::debug!( "schema registry client will use url {:?} to connect, the rest failed because: {:?}", @@ -47,30 +43,34 @@ pub fn handle_sr_list(addr: &str) -> Result> { Ok(urls) } +#[derive(Debug, thiserror::Error)] +pub enum WireFormatError { + #[error("fail to match a magic byte of 0")] + NoMagic, + #[error("fail to read 4-byte schema ID")] + NoSchemaId, + #[error("failed to parse message indexes")] + ParseMessageIndexes, +} + /// extract the magic number and `schema_id` at the front of payload /// /// 0 -> magic number /// 1-4 -> schema id /// 5-... -> message payload -pub(crate) fn extract_schema_id(payload: &[u8]) -> Result<(i32, &[u8])> { - let header_len = 5; - - if payload.len() < header_len { - return Err(RwError::from(InternalError(format!( - "confluent kafka message need 5 bytes header, but payload len is {}", - payload.len() - )))); - } - let magic = payload[0]; - let schema_id = BigEndian::read_i32(&payload[1..5]); +pub(crate) fn extract_schema_id(payload: &[u8]) -> Result<(i32, &[u8]), WireFormatError> { + use byteorder::{BigEndian, ReadBytesExt as _}; - if magic != 0 { - return Err(RwError::from(InternalError( - "confluent kafka message must have a zero magic byte".to_owned(), - ))); + let mut cursor = payload; + if !cursor.read_u8().is_ok_and(|magic| magic == 0) { + return Err(WireFormatError::NoMagic); } - Ok((schema_id, &payload[header_len..])) + let schema_id = cursor + .read_i32::() + .map_err(|_| WireFormatError::NoSchemaId)?; + + Ok((schema_id, cursor)) } pub(crate) struct SchemaRegistryCtx { @@ -80,11 +80,21 @@ pub(crate) struct SchemaRegistryCtx { pub path: Vec, } +#[derive(Debug, thiserror::Error)] +pub enum RequestError { + #[error("confluent registry send req error: {0}")] + Send(#[source] reqwest::Error), + #[error("confluent registry parse resp error: {0}")] + Json(#[source] reqwest::Error), + #[error(transparent)] + Unsuccessful(ErrorResp), +} + pub(crate) async fn req_inner( ctx: Arc, mut url: Url, method: Method, -) -> Result +) -> Result where T: DeserializeOwned + Send + Sync + 'static, { @@ -101,35 +111,17 @@ where request(request_builder).await } -async fn request(req: reqwest::RequestBuilder) -> Result +async fn request(req: reqwest::RequestBuilder) -> Result where T: DeserializeOwned, { - let res = req.send().await.map_err(|e| { - RwError::from(ProtocolError(format!( - "confluent registry send req error {}", - e - ))) - })?; + let res = req.send().await.map_err(RequestError::Send)?; let status = res.status(); if status.is_success() { - res.json().await.map_err(|e| { - RwError::from(ProtocolError(format!( - "confluent registry parse resp error {}", - e - ))) - }) + res.json().await.map_err(RequestError::Json) } else { - let res = res.json::().await.map_err(|e| { - RwError::from(ProtocolError(format!( - "confluent registry resp error {}", - e - ))) - })?; - Err(RwError::from(ProtocolError(format!( - "confluent registry resp error, code: {}, msg {}", - res.error_code, res.message - )))) + let res = res.json().await.map_err(RequestError::Json)?; + Err(RequestError::Unsuccessful(res)) } } @@ -181,18 +173,14 @@ pub struct GetBySubjectResp { pub references: Vec, } -#[derive(Debug, Deserialize)] -struct ErrorResp { +/// +#[derive(Debug, Deserialize, thiserror::Error)] +#[error("confluent schema registry error {error_code}: {message}")] +pub struct ErrorResp { error_code: i32, message: String, } -#[derive(Debug)] -enum ReqResp { - Succeed(T), - Failed(ErrorResp), -} - #[cfg(test)] mod test { use super::super::handle_sr_list; diff --git a/src/connector/src/sink/catalog/desc.rs b/src/connector/src/sink/catalog/desc.rs index 513a39ae90e31..37d35e1d7edde 100644 --- a/src/connector/src/sink/catalog/desc.rs +++ b/src/connector/src/sink/catalog/desc.rs @@ -67,6 +67,9 @@ pub struct SinkDesc { /// Id of the target table for sink into table. pub target_table: Option, + + /// See the same name field in `SinkWriterParam`. + pub extra_partition_col_idx: Option, } impl SinkDesc { @@ -123,6 +126,7 @@ impl SinkDesc { db_name: self.db_name.clone(), sink_from_name: self.sink_from_name.clone(), target_table: self.target_table.map(|table_id| table_id.table_id()), + extra_partition_col_idx: self.extra_partition_col_idx.map(|idx| idx as u64), } } } diff --git a/src/connector/src/sink/catalog/mod.rs b/src/connector/src/sink/catalog/mod.rs index be4f94fa0706b..e6a654f75a5fd 100644 --- a/src/connector/src/sink/catalog/mod.rs +++ b/src/connector/src/sink/catalog/mod.rs @@ -20,6 +20,7 @@ use anyhow::anyhow; use itertools::Itertools; use risingwave_common::catalog::{ ColumnCatalog, ConnectionId, DatabaseId, Field, Schema, SchemaId, TableId, UserId, + OBJECT_ID_PLACEHOLDER, }; use risingwave_common::util::epoch::Epoch; use risingwave_common::util::sort_util::ColumnOrder; @@ -43,7 +44,7 @@ impl SinkId { /// Sometimes the id field is filled later, we use this value for better debugging. pub const fn placeholder() -> Self { SinkId { - sink_id: u32::MAX - 1, + sink_id: OBJECT_ID_PLACEHOLDER, } } @@ -204,7 +205,12 @@ impl TryFrom for SinkFormatDesc { F::Plain => SinkFormat::AppendOnly, F::Upsert => SinkFormat::Upsert, F::Debezium => SinkFormat::Debezium, - f @ (F::Unspecified | F::Native | F::DebeziumMongo | F::Maxwell | F::Canal) => { + f @ (F::Unspecified + | F::Native + | F::DebeziumMongo + | F::Maxwell + | F::Canal + | F::None) => { return Err(SinkError::Config(anyhow!( "sink format unsupported: {}", f.as_str_name() @@ -216,7 +222,7 @@ impl TryFrom for SinkFormatDesc { E::Protobuf => SinkEncode::Protobuf, E::Template => SinkEncode::Template, E::Avro => SinkEncode::Avro, - e @ (E::Unspecified | E::Native | E::Csv | E::Bytes) => { + e @ (E::Unspecified | E::Native | E::Csv | E::Bytes | E::None) => { return Err(SinkError::Config(anyhow!( "sink encode unsupported: {}", e.as_str_name() diff --git a/src/connector/src/sink/clickhouse.rs b/src/connector/src/sink/clickhouse.rs index 123739f4a5618..4e21659dae49e 100644 --- a/src/connector/src/sink/clickhouse.rs +++ b/src/connector/src/sink/clickhouse.rs @@ -26,9 +26,11 @@ use serde::ser::{SerializeSeq, SerializeStruct}; use serde::Serialize; use serde_derive::Deserialize; use serde_with::serde_as; +use thiserror_ext::AsReport; use with_options::WithOptions; use super::{DummySinkCommitCoordinator, SinkWriterParam}; +use crate::error::ConnectorResult; use crate::sink::catalog::desc::SinkDesc; use crate::sink::log_store::DeliveryFutureManagerAddFuture; use crate::sink::writer::{ @@ -131,7 +133,7 @@ impl ClickHouseEngine { const POOL_IDLE_TIMEOUT: Duration = Duration::from_secs(5); impl ClickHouseCommon { - pub(crate) fn build_client(&self) -> anyhow::Result { + pub(crate) fn build_client(&self) -> ConnectorResult { use hyper_tls::HttpsConnector; let https = HttpsConnector::new(); @@ -260,9 +262,11 @@ impl ClickHouseSink { ) -> Result<()> { let is_match = match fields_type { risingwave_common::types::DataType::Boolean => Ok(ck_column.r#type.contains("Bool")), - risingwave_common::types::DataType::Int16 => { - Ok(ck_column.r#type.contains("UInt16") | ck_column.r#type.contains("Int16")) - } + risingwave_common::types::DataType::Int16 => Ok(ck_column.r#type.contains("UInt16") + | ck_column.r#type.contains("Int16") + // Allow Int16 to be pushed to Enum16, they share an encoding and value range + // No special care is taken to ensure values are valid. + | ck_column.r#type.contains("Enum16")), risingwave_common::types::DataType::Int32 => { Ok(ck_column.r#type.contains("UInt32") | ck_column.r#type.contains("Int32")) } @@ -436,7 +440,7 @@ impl ClickHouseSinkWriter { .next() .ok_or_else(|| SinkError::ClickHouse("must have next".to_string()))? .parse::() - .map_err(|e| SinkError::ClickHouse(format!("clickhouse sink error {}", e)))? + .map_err(|e| SinkError::ClickHouse(e.to_report_string()))? } else { 0_u8 }; @@ -455,7 +459,7 @@ impl ClickHouseSinkWriter { .first() .ok_or_else(|| SinkError::ClickHouse("must have next".to_string()))? .parse::() - .map_err(|e| SinkError::ClickHouse(format!("clickhouse sink error {}", e)))?; + .map_err(|e| SinkError::ClickHouse(e.to_report_string()))?; if length > 38 { return Err(SinkError::ClickHouse( @@ -467,7 +471,7 @@ impl ClickHouseSinkWriter { .last() .ok_or_else(|| SinkError::ClickHouse("must have next".to_string()))? .parse::() - .map_err(|e| SinkError::ClickHouse(format!("clickhouse sink error {}", e)))?; + .map_err(|e| SinkError::ClickHouse(e.to_report_string()))?; (length, scale) } else { (0_u8, 0_u8) diff --git a/src/connector/src/sink/coordinate.rs b/src/connector/src/sink/coordinate.rs index 9b771c1eec98f..f2bc4f4b3b362 100644 --- a/src/connector/src/sink/coordinate.rs +++ b/src/connector/src/sink/coordinate.rs @@ -18,9 +18,10 @@ use anyhow::anyhow; use risingwave_common::array::StreamChunk; use risingwave_common::buffer::Bitmap; use risingwave_pb::connector_service::SinkMetadata; -use risingwave_rpc_client::{CoordinatorStreamHandle, SinkCoordinationRpcClient}; +use risingwave_rpc_client::CoordinatorStreamHandle; use tracing::warn; +use super::SinkCoordinationRpcClientEnum; use crate::sink::writer::SinkWriter; use crate::sink::{Result, SinkError, SinkParam}; @@ -32,19 +33,14 @@ pub struct CoordinatedSinkWriter>> CoordinatedSinkWriter { pub async fn new( - client: SinkCoordinationRpcClient, + client: SinkCoordinationRpcClientEnum, param: SinkParam, vnode_bitmap: Bitmap, inner: W, ) -> Result { Ok(Self { epoch: 0, - coordinator_stream_handle: CoordinatorStreamHandle::new( - client, - param.to_proto(), - vnode_bitmap, - ) - .await?, + coordinator_stream_handle: client.new_stream_handle(param, vnode_bitmap).await?, inner, }) } diff --git a/src/connector/src/sink/deltalake.rs b/src/connector/src/sink/deltalake.rs index 0ea787d2735d8..0b9252aca468a 100644 --- a/src/connector/src/sink/deltalake.rs +++ b/src/connector/src/sink/deltalake.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use deltalake::kernel::{Action, Add, DataType as DeltaLakeDataType, PrimitiveType, StructType}; use deltalake::protocol::{DeltaOperation, SaveMode}; @@ -26,9 +26,9 @@ use deltalake::table::builder::s3_storage_options::{ use deltalake::writer::{DeltaWriter, RecordBatchWriter}; use deltalake::DeltaTable; use risingwave_common::array::{to_deltalake_record_batch_with_schema, StreamChunk}; +use risingwave_common::bail; use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::Schema; -use risingwave_common::error::anyhow_error; use risingwave_common::types::DataType; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_pb::connector_service::sink_metadata::Metadata::Serialized; @@ -268,7 +268,7 @@ impl Sink for DeltaLakeSink { .await, self.param.clone(), writer_param.vnode_bitmap.ok_or_else(|| { - SinkError::Remote(anyhow_error!( + SinkError::Remote(anyhow!( "sink needs coordination should not have singleton input" )) })?, @@ -370,7 +370,8 @@ impl DeltaLakeSinkWriter { async fn write(&mut self, chunk: StreamChunk) -> Result<()> { let a = to_deltalake_record_batch_with_schema(self.dl_schema.clone(), &chunk) - .map_err(|err| SinkError::DeltaLake(anyhow!("convert record batch error: {}", err)))?; + .context("convert record batch error") + .map_err(SinkError::DeltaLake)?; self.writer.write(a).await?; Ok(()) } @@ -382,7 +383,8 @@ fn convert_schema(schema: &StructType) -> Result TryFrom<&'a DeltaLakeWriteResult> for SinkMetadata { type Error = SinkError; fn try_from(value: &'a DeltaLakeWriteResult) -> std::prelude::v1::Result { - let metadata = serde_json::to_vec(&value.adds).map_err(|e| -> SinkError { - anyhow!("Can't serialized deltalake sink metadata: {}", e).into() - })?; + let metadata = + serde_json::to_vec(&value.adds).context("cannot serialize deltalake sink metadata")?; Ok(SinkMetadata { metadata: Some(Serialized(SerializedMetadata { metadata })), }) @@ -497,13 +498,11 @@ impl<'a> TryFrom<&'a DeltaLakeWriteResult> for SinkMetadata { impl DeltaLakeWriteResult { fn try_from(value: &SinkMetadata) -> Result { if let Some(Serialized(v)) = &value.metadata { - let adds = - serde_json::from_slice::>(&v.metadata).map_err(|e| -> SinkError { - anyhow!("Can't deserialize deltalake sink metadata: {}", e).into() - })?; + let adds = serde_json::from_slice::>(&v.metadata) + .context("Can't deserialize deltalake sink metadata")?; Ok(DeltaLakeWriteResult { adds }) } else { - Err(anyhow!("Can't create deltalake sink write result from empty data!").into()) + bail!("Can't create deltalake sink write result from empty data!") } } } diff --git a/src/connector/src/sink/doris.rs b/src/connector/src/sink/doris.rs index dbe87b4cb51ca..64ab8121aaaa7 100644 --- a/src/connector/src/sink/doris.rs +++ b/src/connector/src/sink/doris.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use base64::engine::general_purpose; use base64::Engine; @@ -31,6 +31,7 @@ use serde::Deserialize; use serde_derive::Serialize; use serde_json::Value; use serde_with::serde_as; +use thiserror_ext::AsReport; use with_options::WithOptions; use super::doris_starrocks_connector::{ @@ -170,7 +171,7 @@ impl DorisSink { Ok(doris_data_type.contains("DATETIME")) } risingwave_common::types::DataType::Timestamptz => Err(SinkError::Doris( - "doris can not support Timestamptz".to_string(), + "TIMESTAMP WITH TIMEZONE is not supported for Doris sink as Doris doesn't store time values with timezone information. Please convert to TIMESTAMP first.".to_string(), )), risingwave_common::types::DataType::Interval => Err(SinkError::Doris( "doris can not support Interval".to_string(), @@ -326,8 +327,9 @@ impl DorisSinkWriter { DORIS_DELETE_SIGN.to_string(), Value::String("0".to_string()), ); - let row_json_string = serde_json::to_string(&row_json_value) - .map_err(|e| SinkError::Doris(format!("Json derialize error {:?}", e)))?; + let row_json_string = serde_json::to_string(&row_json_value).map_err(|e| { + SinkError::Doris(format!("Json derialize error: {}", e.as_report())) + })?; self.client .as_mut() .ok_or_else(|| { @@ -342,8 +344,9 @@ impl DorisSinkWriter { DORIS_DELETE_SIGN.to_string(), Value::String("1".to_string()), ); - let row_json_string = serde_json::to_string(&row_json_value) - .map_err(|e| SinkError::Doris(format!("Json derialize error {:?}", e)))?; + let row_json_string = serde_json::to_string(&row_json_value).map_err(|e| { + SinkError::Doris(format!("Json derialize error: {}", e.as_report())) + })?; self.client .as_mut() .ok_or_else(|| { @@ -359,8 +362,9 @@ impl DorisSinkWriter { DORIS_DELETE_SIGN.to_string(), Value::String("0".to_string()), ); - let row_json_string = serde_json::to_string(&row_json_value) - .map_err(|e| SinkError::Doris(format!("Json derialize error {:?}", e)))?; + let row_json_string = serde_json::to_string(&row_json_value).map_err(|e| { + SinkError::Doris(format!("Json derialize error: {}", e.as_report())) + })?; self.client .as_mut() .ok_or_else(|| { @@ -471,12 +475,9 @@ impl DorisSchemaClient { } else { raw_bytes }; - let schema: DorisSchema = serde_json::from_str(&json_data).map_err(|err| { - SinkError::DorisStarrocksConnect(anyhow::anyhow!( - "Can't get schema from json {:?}", - err - )) - })?; + let schema: DorisSchema = serde_json::from_str(&json_data) + .context("Can't get schema from json") + .map_err(SinkError::DorisStarrocksConnect)?; Ok(schema) } } diff --git a/src/connector/src/sink/doris_starrocks_connector.rs b/src/connector/src/sink/doris_starrocks_connector.rs index 550572a2b4bcc..8a77f1a13cf1b 100644 --- a/src/connector/src/sink/doris_starrocks_connector.rs +++ b/src/connector/src/sink/doris_starrocks_connector.rs @@ -16,6 +16,7 @@ use core::mem; use core::time::Duration; use std::collections::HashMap; +use anyhow::Context; use base64::engine::general_purpose; use base64::Engine; use bytes::{BufMut, Bytes, BytesMut}; @@ -140,6 +141,14 @@ impl HeaderBuilder { self } + pub fn set_partial_update(mut self, partial_update: Option) -> Self { + self.header.insert( + "partial_update".to_string(), + partial_update.unwrap_or_else(|| "false".to_string()), + ); + self + } + pub fn build(self) -> HashMap { self.header } @@ -196,12 +205,8 @@ impl InserterInnerBuilder { )) })? .to_str() - .map_err(|err| { - SinkError::DorisStarrocksConnect(anyhow::anyhow!( - "Can't get doris BE url in header {:?}", - err - )) - })? + .context("Can't get doris BE url in header") + .map_err(SinkError::DorisStarrocksConnect)? } else { return Err(SinkError::DorisStarrocksConnect(anyhow::anyhow!( "Can't get doris BE url", diff --git a/src/connector/src/sink/encoder/avro.rs b/src/connector/src/sink/encoder/avro.rs index d63ab69951b08..924beb281eda7 100644 --- a/src/connector/src/sink/encoder/avro.rs +++ b/src/connector/src/sink/encoder/avro.rs @@ -20,6 +20,7 @@ use risingwave_common::catalog::Schema; use risingwave_common::row::Row; use risingwave_common::types::{DataType, DatumRef, ScalarRefImpl, StructType}; use risingwave_common::util::iter_util::{ZipEqDebug, ZipEqFast}; +use thiserror_ext::AsReport; use super::{FieldEncodeError, Result as SinkResult, RowEncoder, SerTo}; @@ -134,7 +135,7 @@ impl SerTo> for AvroEncoded { ))); }; let raw = apache_avro::to_avro_datum(&self.schema, self.value) - .map_err(|e| crate::sink::SinkError::Encode(e.to_string()))?; + .map_err(|e| crate::sink::SinkError::Encode(e.to_report_string()))?; let mut buf = Vec::with_capacity(1 + 4 + raw.len()); buf.put_u8(0); buf.put_i32(schema_id); diff --git a/src/connector/src/sink/encoder/json.rs b/src/connector/src/sink/encoder/json.rs index b4d2de84c0069..eb5c7b129385d 100644 --- a/src/connector/src/sink/encoder/json.rs +++ b/src/connector/src/sink/encoder/json.rs @@ -15,6 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; +use anyhow::Context; use base64::engine::general_purpose; use base64::Engine as _; use chrono::{Datelike, NaiveDateTime, Timelike}; @@ -26,6 +27,7 @@ use risingwave_common::row::Row; use risingwave_common::types::{DataType, DatumRef, Decimal, JsonbVal, ScalarRefImpl, ToText}; use risingwave_common::util::iter_util::ZipEqDebug; use serde_json::{json, Map, Value}; +use thiserror_ext::AsReport; use super::{ CustomJsonType, DateHandlingMode, KafkaConnectParams, KafkaConnectParamsRef, Result, @@ -96,6 +98,24 @@ impl JsonEncoder { } } + pub fn new_with_starrocks( + schema: Schema, + col_indices: Option>, + timestamp_handling_mode: TimestampHandlingMode, + map: HashMap, + ) -> Self { + Self { + schema, + col_indices, + time_handling_mode: TimeHandlingMode::Milli, + date_handling_mode: DateHandlingMode::String, + timestamp_handling_mode, + timestamptz_handling_mode: TimestamptzHandlingMode::UtcWithoutSuffix, + custom_json_type: CustomJsonType::StarRocks(map), + kafka_connect: None, + } + } + pub fn with_kafka_connect(self, kafka_connect: KafkaConnectParams) -> Self { Self { kafka_connect: Some(Arc::new(kafka_connect)), @@ -134,7 +154,7 @@ impl RowEncoder for JsonEncoder { self.time_handling_mode, &self.custom_json_type, ) - .map_err(|e| SinkError::Encode(e.to_string()))?; + .map_err(|e| SinkError::Encode(e.to_report_string()))?; mappings.insert(key, value); } @@ -203,7 +223,7 @@ fn datum_to_json_object( json!(v) } (DataType::Decimal, ScalarRefImpl::Decimal(mut v)) => match custom_json_type { - CustomJsonType::Doris(map) => { + CustomJsonType::Doris(map) | CustomJsonType::StarRocks(map) => { if !matches!(v, Decimal::Normalized(_)) { return Err(ArrayError::internal( "doris/starrocks can't support decimal Inf, -Inf, Nan".to_string(), @@ -270,8 +290,10 @@ fn datum_to_json_object( json!(v.as_iso_8601()) } (DataType::Jsonb, ScalarRefImpl::Jsonb(jsonb_ref)) => match custom_json_type { - CustomJsonType::Es => JsonbVal::from(jsonb_ref).take(), - CustomJsonType::Doris(_) | CustomJsonType::None => json!(jsonb_ref.to_string()), + CustomJsonType::Es | CustomJsonType::StarRocks(_) => JsonbVal::from(jsonb_ref).take(), + CustomJsonType::Doris(_) | CustomJsonType::None => { + json!(jsonb_ref.to_string()) + } }, (DataType::List(datatype), ScalarRefImpl::List(list_ref)) => { let elems = list_ref.iter(); @@ -311,9 +333,14 @@ fn datum_to_json_object( )?; map.insert(sub_field.name.clone(), value); } - Value::String(serde_json::to_string(&map).map_err(|err| { - ArrayError::internal(format!("Json to string err{:?}", err)) - })?) + Value::String( + serde_json::to_string(&map).context("failed to serialize into JSON")?, + ) + } + CustomJsonType::StarRocks(_) => { + return Err(ArrayError::internal( + "starrocks can't support struct".to_string(), + )); } CustomJsonType::Es | CustomJsonType::None => { let mut map = Map::with_capacity(st.len()); diff --git a/src/connector/src/sink/encoder/mod.rs b/src/connector/src/sink/encoder/mod.rs index 83c28ce4b4a5a..3c76803c8e0f1 100644 --- a/src/connector/src/sink/encoder/mod.rs +++ b/src/connector/src/sink/encoder/mod.rs @@ -142,6 +142,8 @@ pub enum CustomJsonType { Doris(HashMap), // Es's json need jsonb is struct Es, + // starrocks' need jsonb is struct + StarRocks(HashMap), None, } diff --git a/src/connector/src/sink/formatter/mod.rs b/src/connector/src/sink/formatter/mod.rs index 74c6d54151133..cb36bda793709 100644 --- a/src/connector/src/sink/formatter/mod.rs +++ b/src/connector/src/sink/formatter/mod.rs @@ -126,7 +126,7 @@ impl SinkFormatterImpl { let descriptor = crate::schema::protobuf::fetch_descriptor(&format_desc.options, None) .await - .map_err(|e| SinkError::Config(anyhow!("{e:?}")))?; + .map_err(|e| SinkError::Config(anyhow!(e)))?; let val_encoder = ProtoEncoder::new(schema, None, descriptor)?; let formatter = AppendOnlyFormatter::new(key_encoder, val_encoder); Ok(SinkFormatterImpl::AppendOnlyProto(formatter)) @@ -240,7 +240,7 @@ impl SinkFormatterImpl { let (key_schema, val_schema) = crate::schema::avro::fetch_schema(&format_desc.options, topic) .await - .map_err(|e| SinkError::Config(anyhow!("{e:?}")))?; + .map_err(|e| SinkError::Config(anyhow!(e)))?; let key_encoder = AvroEncoder::new( schema.clone(), Some(pk_indices), diff --git a/src/connector/src/sink/iceberg/jni_catalog.rs b/src/connector/src/sink/iceberg/jni_catalog.rs new file mode 100644 index 0000000000000..d88a63d398c65 --- /dev/null +++ b/src/connector/src/sink/iceberg/jni_catalog.rs @@ -0,0 +1,186 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! This module provide jni catalog. + +use std::collections::HashMap; +use std::sync::Arc; + +use anyhow::Context; +use async_trait::async_trait; +use icelake::catalog::models::{CommitTableRequest, CommitTableResponse, LoadTableResult}; +use icelake::catalog::{ + BaseCatalogConfig, Catalog, CatalogRef, IcebergTableIoArgs, OperatorCreator, UpdateTable, +}; +use icelake::{ErrorKind, Table, TableIdentifier}; +use jni::objects::{GlobalRef, JObject}; +use jni::JavaVM; +use risingwave_jni_core::call_method; +use risingwave_jni_core::jvm_runtime::{execute_with_jni_env, jobj_to_str, JVM}; + +use crate::error::ConnectorResult; + +pub struct JniCatalog { + java_catalog: GlobalRef, + jvm: &'static JavaVM, + config: BaseCatalogConfig, +} + +#[async_trait] +impl Catalog for JniCatalog { + fn name(&self) -> &str { + &self.config.name + } + + async fn load_table(self: Arc, table_name: &TableIdentifier) -> icelake::Result

{ + execute_with_jni_env(self.jvm, |env| { + let table_name_str = table_name.to_string(); + + let table_name_jstr = env.new_string(&table_name_str).unwrap(); + + let result_json = + call_method!(env, self.java_catalog.as_obj(), {String loadTable(String)}, + &table_name_jstr) + .with_context(|| format!("Failed to load iceberg table: {table_name_str}"))?; + + let rust_json_str = jobj_to_str(env, result_json)?; + + let resp: LoadTableResult = serde_json::from_str(&rust_json_str)?; + + let metadata_location = resp.metadata_location.clone().ok_or_else(|| { + icelake::Error::new( + ErrorKind::IcebergFeatureUnsupported, + "Loading uncommitted table is not supported!", + ) + })?; + + tracing::info!("Table metadata location of {table_name} is {metadata_location}"); + + let table_metadata = resp.table_metadata()?; + + let iceberg_io_args = IcebergTableIoArgs::builder_from_path(&table_metadata.location)? + .with_args(self.config.table_io_configs.iter()) + .build()?; + let table_op = iceberg_io_args.create()?; + + Ok(Table::builder_from_catalog( + table_op, + self.clone(), + table_metadata, + table_name.clone(), + ) + .build()?) + }) + .map_err(|e| { + icelake::Error::new(ErrorKind::Unexpected, "Failed to load iceberg table.") + .set_source(e) + }) + } + + async fn update_table(self: Arc, update_table: &UpdateTable) -> icelake::Result
{ + execute_with_jni_env(self.jvm, |env| { + let request_str = serde_json::to_string(&CommitTableRequest::try_from(update_table)?)?; + + let request_jni_str = env.new_string(&request_str).with_context(|| { + format!("Failed to create jni string from request json: {request_str}.") + })?; + + let result_json = + call_method!(env, self.java_catalog.as_obj(), {String updateTable(String)}, + &request_jni_str) + .with_context(|| { + format!( + "Failed to update iceberg table: {}", + update_table.table_name() + ) + })?; + + let rust_json_str = jobj_to_str(env, result_json)?; + + let response: CommitTableResponse = serde_json::from_str(&rust_json_str)?; + + tracing::info!( + "Table metadata location of {} is {}", + update_table.table_name(), + response.metadata_location + ); + + let table_metadata = response.metadata()?; + + let args = IcebergTableIoArgs::builder_from_path(&table_metadata.location)? + .with_args(self.config.table_io_configs.iter()) + .build()?; + let table_op = args.create()?; + + Ok(Table::builder_from_catalog( + table_op, + self.clone(), + table_metadata, + update_table.table_name().clone(), + ) + .build()?) + }) + .map_err(|e| { + icelake::Error::new(ErrorKind::Unexpected, "Failed to update iceberg table.") + .set_source(e) + }) + } +} + +impl JniCatalog { + pub fn build( + base_config: BaseCatalogConfig, + name: impl ToString, + catalog_impl: impl ToString, + java_catalog_props: HashMap, + ) -> ConnectorResult { + let jvm = JVM.get_or_init()?; + + execute_with_jni_env(jvm, |env| { + // Convert props to string array + let props = env.new_object_array( + (java_catalog_props.len() * 2) as i32, + "java/lang/String", + JObject::null(), + )?; + for (i, (key, value)) in java_catalog_props.iter().enumerate() { + let key_j_str = env.new_string(key)?; + let value_j_str = env.new_string(value)?; + env.set_object_array_element(&props, i as i32 * 2, key_j_str)?; + env.set_object_array_element(&props, i as i32 * 2 + 1, value_j_str)?; + } + + let jni_catalog_wrapper = env + .call_static_method( + "com/risingwave/connector/catalog/JniCatalogWrapper", + "create", + "(Ljava/lang/String;Ljava/lang/String;[Ljava/lang/String;)Lcom/risingwave/connector/catalog/JniCatalogWrapper;", + &[ + (&env.new_string(name.to_string()).unwrap()).into(), + (&env.new_string(catalog_impl.to_string()).unwrap()).into(), + (&props).into(), + ], + )?; + + let jni_catalog = env.new_global_ref(jni_catalog_wrapper.l().unwrap())?; + + Ok(Arc::new(Self { + java_catalog: jni_catalog, + jvm, + config: base_config, + }) as CatalogRef) + }) + .map_err(Into::into) + } +} diff --git a/src/connector/src/sink/iceberg/mock_catalog.rs b/src/connector/src/sink/iceberg/mock_catalog.rs new file mode 100644 index 0000000000000..f9d60965b1d30 --- /dev/null +++ b/src/connector/src/sink/iceberg/mock_catalog.rs @@ -0,0 +1,237 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::sync::Arc; + +use async_trait::async_trait; +use icelake::catalog::{Catalog, UpdateTable}; +use icelake::types::{Field, PartitionField, Schema, Struct, TableMetadata}; +use icelake::{Table, TableIdentifier}; +use opendal::services::Memory; +use opendal::Operator; + +/// A mock catalog for iceberg used for plan test. +pub struct MockCatalog; + +impl MockCatalog { + const RANGE_TABLE: &'static str = "range_table"; + const SPARSE_TABLE: &'static str = "sparse_table"; + + fn sparse_table(self: &Arc) -> Table { + Table::builder_from_catalog( + { + let mut builder = Memory::default(); + builder.root("/tmp"); + Operator::new(builder).unwrap().finish() + }, + self.clone(), + TableMetadata { + format_version: icelake::types::TableFormatVersion::V2, + table_uuid: "1".to_string(), + location: "1".to_string(), + last_sequence_number: 1, + last_updated_ms: 1, + last_column_id: 1, + schemas: vec![Schema::new( + 1, + None, + Struct::new(vec![ + Field::required( + 1, + "v1", + icelake::types::Any::Primitive(icelake::types::Primitive::Int), + ) + .into(), + Field::required( + 2, + "v2", + icelake::types::Any::Primitive(icelake::types::Primitive::Long), + ) + .into(), + Field::required( + 3, + "v3", + icelake::types::Any::Primitive(icelake::types::Primitive::String), + ) + .into(), + Field::required( + 4, + "v4", + icelake::types::Any::Primitive(icelake::types::Primitive::Time), + ) + .into(), + ]), + )], + current_schema_id: 1, + partition_specs: vec![icelake::types::PartitionSpec { + spec_id: 1, + fields: vec![ + PartitionField { + source_column_id: 1, + partition_field_id: 5, + transform: icelake::types::Transform::Identity, + name: "f1".to_string(), + }, + PartitionField { + source_column_id: 2, + partition_field_id: 6, + transform: icelake::types::Transform::Bucket(1), + name: "f2".to_string(), + }, + PartitionField { + source_column_id: 3, + partition_field_id: 7, + transform: icelake::types::Transform::Truncate(1), + name: "f3".to_string(), + }, + PartitionField { + source_column_id: 4, + partition_field_id: 8, + transform: icelake::types::Transform::Void, + name: "f4".to_string(), + }, + ], + }], + default_spec_id: 1, + last_partition_id: 1, + properties: None, + current_snapshot_id: None, + snapshots: None, + snapshot_log: None, + metadata_log: None, + sort_orders: vec![], + default_sort_order_id: 0, + refs: HashMap::new(), + }, + TableIdentifier::new(vec![Self::SPARSE_TABLE]).unwrap(), + ) + .build() + .unwrap() + } + + fn range_table(self: &Arc) -> Table { + Table::builder_from_catalog( + { + let mut builder = Memory::default(); + builder.root("/tmp"); + Operator::new(builder).unwrap().finish() + }, + self.clone(), + TableMetadata { + format_version: icelake::types::TableFormatVersion::V2, + table_uuid: "1".to_string(), + location: "1".to_string(), + last_sequence_number: 1, + last_updated_ms: 1, + last_column_id: 1, + schemas: vec![Schema::new( + 1, + None, + Struct::new(vec![ + Field::required( + 1, + "v1", + icelake::types::Any::Primitive(icelake::types::Primitive::Date), + ) + .into(), + Field::required( + 2, + "v2", + icelake::types::Any::Primitive(icelake::types::Primitive::Timestamp), + ) + .into(), + Field::required( + 3, + "v3", + icelake::types::Any::Primitive(icelake::types::Primitive::Timestampz), + ) + .into(), + Field::required( + 4, + "v4", + icelake::types::Any::Primitive(icelake::types::Primitive::Timestampz), + ) + .into(), + ]), + )], + current_schema_id: 1, + partition_specs: vec![icelake::types::PartitionSpec { + spec_id: 1, + fields: vec![ + PartitionField { + source_column_id: 1, + partition_field_id: 7, + transform: icelake::types::Transform::Year, + name: "f4".to_string(), + }, + PartitionField { + source_column_id: 2, + partition_field_id: 8, + transform: icelake::types::Transform::Month, + name: "f5".to_string(), + }, + PartitionField { + source_column_id: 3, + partition_field_id: 9, + transform: icelake::types::Transform::Day, + name: "f6".to_string(), + }, + PartitionField { + source_column_id: 4, + partition_field_id: 10, + transform: icelake::types::Transform::Hour, + name: "f7".to_string(), + }, + ], + }], + default_spec_id: 1, + last_partition_id: 1, + properties: None, + current_snapshot_id: None, + snapshots: None, + snapshot_log: None, + metadata_log: None, + sort_orders: vec![], + default_sort_order_id: 0, + refs: HashMap::new(), + }, + TableIdentifier::new(vec![Self::RANGE_TABLE]).unwrap(), + ) + .build() + .unwrap() + } +} + +#[async_trait] +impl Catalog for MockCatalog { + fn name(&self) -> &str { + "mock" + } + + // Mock catalog load mock table according to table_name, there is 2 kinds of table for test: + // 1. sparse partition table + // 2. range partition table + async fn load_table(self: Arc, table_name: &TableIdentifier) -> icelake::Result
{ + match table_name.name.as_ref() { + Self::SPARSE_TABLE => Ok(self.sparse_table()), + Self::RANGE_TABLE => Ok(self.range_table()), + _ => unimplemented!("table {} not found", table_name), + } + } + + async fn update_table(self: Arc, _update_table: &UpdateTable) -> icelake::Result
{ + unimplemented!() + } +} diff --git a/src/connector/src/sink/iceberg/mod.rs b/src/connector/src/sink/iceberg/mod.rs index b2cbcef9f6712..374f8a925b1f0 100644 --- a/src/connector/src/sink/iceberg/mod.rs +++ b/src/connector/src/sink/iceberg/mod.rs @@ -12,16 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +mod jni_catalog; +mod mock_catalog; mod prometheus; use std::collections::HashMap; use std::fmt::Debug; use std::ops::Deref; +use std::sync::Arc; -use anyhow::anyhow; -use arrow_schema::{DataType as ArrowDataType, Schema as ArrowSchema, SchemaRef}; +use anyhow::{anyhow, Context}; +use arrow_schema::{ + DataType as ArrowDataType, Field as ArrowField, Fields, Schema as ArrowSchema, SchemaRef, +}; use async_trait::async_trait; -use icelake::catalog::{load_catalog, CATALOG_NAME, CATALOG_TYPE}; +use icelake::catalog::{ + load_catalog, load_iceberg_base_catalog_config, BaseCatalogConfig, CatalogRef, CATALOG_NAME, + CATALOG_TYPE, +}; use icelake::io_v2::input_wrapper::{DeltaWriter, RecordBatchWriter}; use icelake::io_v2::prometheus::{PrometheusWriterBuilder, WriterMetrics}; use icelake::io_v2::{ @@ -31,25 +39,27 @@ use icelake::transaction::Transaction; use icelake::types::{data_file_from_json, data_file_to_json, Any, DataFile}; use icelake::{Table, TableIdentifier}; use itertools::Itertools; -use risingwave_common::array::{to_record_batch_with_schema, Op, StreamChunk}; +use risingwave_common::array::{to_iceberg_record_batch_with_schema, Op, StreamChunk}; +use risingwave_common::bail; use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::Schema; -use risingwave_common::error::anyhow_error; use risingwave_pb::connector_service::sink_metadata::Metadata::Serialized; use risingwave_pb::connector_service::sink_metadata::SerializedMetadata; use risingwave_pb::connector_service::SinkMetadata; use serde::de; use serde_derive::Deserialize; +use thiserror_ext::AsReport; use url::Url; use with_options::WithOptions; +use self::mock_catalog::MockCatalog; use self::prometheus::monitored_base_file_writer::MonitoredBaseFileWriterBuilder; -use self::prometheus::monitored_partition_writer::MonitoredFanoutPartitionedWriterBuilder; use self::prometheus::monitored_position_delete_writer::MonitoredPositionDeleteWriterBuilder; use super::{ Sink, SinkError, SinkWriterParam, SINK_TYPE_APPEND_ONLY, SINK_TYPE_OPTION, SINK_TYPE_UPSERT, }; use crate::deserialize_bool_from_string; +use crate::error::ConnectorResult; use crate::sink::coordinate::CoordinatedSinkWriter; use crate::sink::writer::{LogSinkerOf, SinkWriter, SinkWriterExt}; use crate::sink::{Result, SinkCommitCoordinator, SinkParam}; @@ -57,8 +67,7 @@ use crate::sink::{Result, SinkCommitCoordinator, SinkParam}; /// This iceberg sink is WIP. When it ready, we will change this name to "iceberg". pub const ICEBERG_SINK: &str = "iceberg"; -#[derive(Debug, Clone, Deserialize, WithOptions)] -#[serde(deny_unknown_fields)] +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, WithOptions, Default)] pub struct IcebergConfig { pub connector: String, // Avoid deny unknown field. Must be "iceberg" @@ -71,7 +80,13 @@ pub struct IcebergConfig { pub table_name: String, // Full name of table, must include schema name #[serde(rename = "database.name")] - pub database_name: String, // Use as catalog name. + pub database_name: Option, + // Database name of table + + // Catalog name, can be omitted for storage catalog, but + // must be set for other catalogs. + #[serde(rename = "catalog.name")] + pub catalog_name: Option, // Catalog type supported by iceberg, such as "storage", "rest". // If not set, we use "storage" as default. @@ -102,7 +117,12 @@ pub struct IcebergConfig { deserialize_with = "deserialize_string_seq_from_string" )] pub primary_key: Option>, + + // Props for java catalog props. + #[serde(skip)] + pub java_catalog_props: HashMap, } + pub(crate) fn deserialize_string_seq_from_string<'de, D>( deserializer: D, ) -> std::result::Result>, D::Error> @@ -121,8 +141,9 @@ where impl IcebergConfig { pub fn from_hashmap(values: HashMap) -> Result { - let config = serde_json::from_value::(serde_json::to_value(values).unwrap()) - .map_err(|e| SinkError::Config(anyhow!(e)))?; + let mut config = + serde_json::from_value::(serde_json::to_value(&values).unwrap()) + .map_err(|e| SinkError::Config(anyhow!(e)))?; if config.r#type != SINK_TYPE_APPEND_ONLY && config.r#type != SINK_TYPE_UPSERT { return Err(SinkError::Config(anyhow!( @@ -149,28 +170,61 @@ impl IcebergConfig { } } + if config.catalog_name.is_none() && config.catalog_type.as_deref() != Some("storage") { + return Err(SinkError::Config(anyhow!( + "catalog.name must be set for non-storage catalog" + ))); + } + + // All configs start with "catalog." will be treated as java configs. + config.java_catalog_props = values + .iter() + .filter(|(k, _v)| { + k.starts_with("catalog.") + && k != &"catalog.uri" + && k != &"catalog.type" + && k != &"catalog.name" + }) + .map(|(k, v)| (k[8..].to_string(), v.to_string())) + .collect(); + Ok(config) } + fn catalog_type(&self) -> &str { + self.catalog_type.as_deref().unwrap_or("storage") + } + + fn catalog_name(&self) -> String { + self.catalog_name + .as_ref() + .map(|s| s.to_string()) + .unwrap_or_else(|| "risingwave".to_string()) + } + + fn full_table_name(&self) -> Result { + let ret = if let Some(database_name) = &self.database_name { + TableIdentifier::new(vec![database_name, &self.table_name]) + } else { + TableIdentifier::new(vec![&self.table_name]) + }; + + ret.context("Failed to create table identifier") + .map_err(|e| SinkError::Iceberg(anyhow!(e))) + } + fn build_iceberg_configs(&self) -> Result> { let mut iceberg_configs = HashMap::new(); - let catalog_type = self - .catalog_type - .as_deref() - .unwrap_or("storage") - .to_string(); + let catalog_type = self.catalog_type().to_string(); iceberg_configs.insert(CATALOG_TYPE.to_string(), catalog_type.clone()); - iceberg_configs.insert( - CATALOG_NAME.to_string(), - self.database_name.clone().to_string(), - ); + iceberg_configs.insert(CATALOG_NAME.to_string(), self.catalog_name()); match catalog_type.as_str() { "storage" => { iceberg_configs.insert( - format!("iceberg.catalog.{}.warehouse", self.database_name), + format!("iceberg.catalog.{}.warehouse", self.catalog_name()), self.path.clone(), ); } @@ -178,13 +232,13 @@ impl IcebergConfig { let uri = self.uri.clone().ok_or_else(|| { SinkError::Iceberg(anyhow!("`catalog.uri` must be set in rest catalog")) })?; - iceberg_configs.insert(format!("iceberg.catalog.{}.uri", self.database_name), uri); + iceberg_configs.insert(format!("iceberg.catalog.{}.uri", self.catalog_name()), uri); } _ => { return Err(SinkError::Iceberg(anyhow!( "Unsupported catalog type: {}, only support `storage` and `rest`", catalog_type - ))) + ))); } } @@ -224,7 +278,11 @@ impl IcebergConfig { }; iceberg_configs.insert("iceberg.table.io.bucket".to_string(), bucket); - iceberg_configs.insert("iceberg.table.io.root".to_string(), root); + + // Only storage catalog should set this. + if catalog_type == "storage" { + iceberg_configs.insert("iceberg.table.io.root".to_string(), root); + } // #TODO // Support load config file iceberg_configs.insert( @@ -234,6 +292,144 @@ impl IcebergConfig { Ok(iceberg_configs) } + + fn build_jni_catalog_configs(&self) -> Result<(BaseCatalogConfig, HashMap)> { + let mut iceberg_configs = HashMap::new(); + + let base_catalog_config = { + let catalog_type = self.catalog_type().to_string(); + + iceberg_configs.insert(CATALOG_TYPE.to_string(), catalog_type.clone()); + iceberg_configs.insert(CATALOG_NAME.to_string(), self.catalog_name()); + + if let Some(region) = &self.region { + iceberg_configs.insert( + "iceberg.table.io.region".to_string(), + region.clone().to_string(), + ); + } + + if let Some(endpoint) = &self.endpoint { + iceberg_configs.insert( + "iceberg.table.io.endpoint".to_string(), + endpoint.clone().to_string(), + ); + } + + iceberg_configs.insert( + "iceberg.table.io.access_key_id".to_string(), + self.access_key.clone().to_string(), + ); + iceberg_configs.insert( + "iceberg.table.io.secret_access_key".to_string(), + self.secret_key.clone().to_string(), + ); + + let (bucket, _) = { + let url = Url::parse(&self.path).map_err(|e| SinkError::Iceberg(anyhow!(e)))?; + let bucket = url + .host_str() + .ok_or_else(|| { + SinkError::Iceberg(anyhow!( + "Invalid s3 path: {}, bucket is missing", + self.path + )) + })? + .to_string(); + let root = url.path().trim_start_matches('/').to_string(); + (bucket, root) + }; + + iceberg_configs.insert("iceberg.table.io.bucket".to_string(), bucket); + // #TODO + // Support load config file + iceberg_configs.insert( + "iceberg.table.io.disable_config_load".to_string(), + "true".to_string(), + ); + + load_iceberg_base_catalog_config(&iceberg_configs)? + }; + + // Prepare jni configs, for details please see https://iceberg.apache.org/docs/latest/aws/ + let mut java_catalog_configs = HashMap::new(); + { + if let Some(uri) = self.uri.as_deref() { + java_catalog_configs.insert("uri".to_string(), uri.to_string()); + } + + java_catalog_configs.insert("warehouse".to_string(), self.path.clone()); + java_catalog_configs.extend(self.java_catalog_props.clone()); + + // Currently we only support s3, so let's set it to s3 + java_catalog_configs.insert( + "io-impl".to_string(), + "org.apache.iceberg.aws.s3.S3FileIO".to_string(), + ); + + if let Some(endpoint) = &self.endpoint { + java_catalog_configs + .insert("s3.endpoint".to_string(), endpoint.clone().to_string()); + } + + java_catalog_configs.insert( + "s3.access-key-id".to_string(), + self.access_key.clone().to_string(), + ); + java_catalog_configs.insert( + "s3.secret-access-key".to_string(), + self.secret_key.clone().to_string(), + ); + } + + Ok((base_catalog_config, java_catalog_configs)) + } + + async fn create_catalog(&self) -> ConnectorResult { + match self.catalog_type() { + "storage" | "rest" => { + let iceberg_configs = self.build_iceberg_configs()?; + let catalog = load_catalog(&iceberg_configs).await?; + Ok(catalog) + } + catalog_type if catalog_type == "hive" || catalog_type == "jdbc" => { + // Create java catalog + let (base_catalog_config, java_catalog_props) = self.build_jni_catalog_configs()?; + let catalog_impl = match catalog_type { + "hive" => "org.apache.iceberg.hive.HiveCatalog", + "jdbc" => "org.apache.iceberg.jdbc.JdbcCatalog", + _ => unreachable!(), + }; + + jni_catalog::JniCatalog::build( + base_catalog_config, + self.catalog_name(), + catalog_impl, + java_catalog_props, + ) + } + "mock" => Ok(Arc::new(MockCatalog {})), + _ => { + bail!( + "Unsupported catalog type: {}, only support `storage`, `rest`, `hive`, `jdbc`", + self.catalog_type() + ) + } + } + } + + pub async fn load_table(&self) -> ConnectorResult
{ + let catalog = self + .create_catalog() + .await + .context("Unable to load iceberg catalog")?; + + let table_id = self + .full_table_name() + .context("Unable to parse table name")?; + + catalog.load_table(&table_id).await.map_err(Into::into) + } } pub struct IcebergSink { @@ -261,16 +457,10 @@ impl Debug for IcebergSink { } impl IcebergSink { - async fn create_table(&self) -> Result
{ - let catalog = load_catalog(&self.config.build_iceberg_configs()?) - .await - .map_err(|e| SinkError::Iceberg(anyhow!("Unable to load iceberg catalog: {e}")))?; - - let table_id = TableIdentifier::new(self.config.table_name.split('.')) - .map_err(|e| SinkError::Iceberg(anyhow!("Unable to parse table name: {e}")))?; - - let table = catalog - .load_table(&table_id) + async fn create_and_validate_table(&self) -> Result
{ + let table = self + .config + .load_table() .await .map_err(|err| SinkError::Iceberg(anyhow!(err)))?; @@ -283,7 +473,8 @@ impl IcebergSink { .try_into() .map_err(|err: icelake::Error| SinkError::Iceberg(anyhow!(err)))?; - try_matches_arrow_schema(&sink_schema, &iceberg_schema)?; + try_matches_arrow_schema(&sink_schema, &iceberg_schema, false) + .map_err(|err| SinkError::Iceberg(anyhow!(err)))?; Ok(table) } @@ -329,12 +520,12 @@ impl Sink for IcebergSink { const SINK_NAME: &'static str = ICEBERG_SINK; async fn validate(&self) -> Result<()> { - let _ = self.create_table().await?; + let _ = self.create_and_validate_table().await?; Ok(()) } async fn new_log_sinker(&self, writer_param: SinkWriterParam) -> Result { - let table = self.create_table().await?; + let table = self.create_and_validate_table().await?; let inner = if let Some(unique_column_ids) = &self.unique_column_ids { IcebergWriter::new_upsert(table, unique_column_ids.clone(), &writer_param).await? } else { @@ -348,7 +539,7 @@ impl Sink for IcebergSink { .await, self.param.clone(), writer_param.vnode_bitmap.ok_or_else(|| { - SinkError::Remote(anyhow_error!( + SinkError::Remote(anyhow!( "sink needs coordination should not have singleton input" )) })?, @@ -359,7 +550,7 @@ impl Sink for IcebergSink { } async fn new_coordinator(&self) -> Result { - let table = self.create_table().await?; + let table = self.create_and_validate_table().await?; let partition_type = table.current_partition_type()?; Ok(IcebergSinkCommitter { @@ -380,6 +571,31 @@ enum IcebergWriterEnum { } impl IcebergWriter { + fn schema_with_extra_partition_col(table: &Table, idx: usize) -> Result { + let schema = table.current_arrow_schema()?; + + let mut fields = schema.fields().to_vec(); + let partition_type = + if let ArrowDataType::Struct(s) = table.current_partition_type()?.try_into()? { + let fields = Fields::from( + s.into_iter() + .enumerate() + .map(|(id, field)| { + ArrowField::new(format!("f{id}"), field.data_type().clone(), true) + }) + .collect::>(), + ); + ArrowDataType::Struct(fields) + } else { + unimplemented!() + }; + fields.insert( + idx, + ArrowField::new("_rw_partition", partition_type, false).into(), + ); + Ok(ArrowSchema::new(fields).into()) + } + pub async fn new_append_only(table: Table, writer_param: &SinkWriterParam) -> Result { let builder_helper = table.builder_helper()?; @@ -391,30 +607,54 @@ impl IcebergWriter { .iceberg_rolling_unflushed_data_file .clone(), )); - let partition_data_file_builder = MonitoredFanoutPartitionedWriterBuilder::new( - builder_helper.fanout_partition_writer_builder(data_file_builder.clone())?, - writer_param.sink_metrics.iceberg_partition_num.clone(), - ); - let dispatch_builder = builder_helper - .dispatcher_writer_builder(partition_data_file_builder, data_file_builder)?; - // wrap a layer with collect write metrics - let prometheus_builder = PrometheusWriterBuilder::new( - dispatch_builder, - WriterMetrics::new( - writer_param.sink_metrics.iceberg_write_qps.deref().clone(), - writer_param - .sink_metrics - .iceberg_write_latency - .deref() - .clone(), - ), - ); - let schema = table.current_arrow_schema()?; - let inner_writer = RecordBatchWriter::new(prometheus_builder.build(&schema).await?); - Ok(Self { - inner_writer: IcebergWriterEnum::AppendOnly(inner_writer), - schema, - }) + if let Some(extra_partition_col_idx) = writer_param.extra_partition_col_idx { + let partition_data_file_builder = builder_helper.precompute_partition_writer_builder( + data_file_builder.clone(), + extra_partition_col_idx, + )?; + let dispatch_builder = builder_helper + .dispatcher_writer_builder(partition_data_file_builder, data_file_builder)?; + let prometheus_builder = PrometheusWriterBuilder::new( + dispatch_builder, + WriterMetrics::new( + writer_param.sink_metrics.iceberg_write_qps.deref().clone(), + writer_param + .sink_metrics + .iceberg_write_latency + .deref() + .clone(), + ), + ); + let schema = Self::schema_with_extra_partition_col(&table, extra_partition_col_idx)?; + let inner_writer = RecordBatchWriter::new(prometheus_builder.build(&schema).await?); + Ok(Self { + inner_writer: IcebergWriterEnum::AppendOnly(inner_writer), + schema, + }) + } else { + let partition_data_file_builder = + builder_helper.fanout_partition_writer_builder(data_file_builder.clone())?; + let dispatch_builder = builder_helper + .dispatcher_writer_builder(partition_data_file_builder, data_file_builder)?; + // wrap a layer with collect write metrics + let prometheus_builder = PrometheusWriterBuilder::new( + dispatch_builder, + WriterMetrics::new( + writer_param.sink_metrics.iceberg_write_qps.deref().clone(), + writer_param + .sink_metrics + .iceberg_write_latency + .deref() + .clone(), + ), + ); + let schema = table.current_arrow_schema()?; + let inner_writer = RecordBatchWriter::new(prometheus_builder.build(&schema).await?); + Ok(Self { + inner_writer: IcebergWriterEnum::AppendOnly(inner_writer), + schema, + }) + } } pub async fn new_upsert( @@ -446,30 +686,55 @@ impl IcebergWriter { equality_delete_builder, unique_column_ids, ); - let partition_delta_builder = MonitoredFanoutPartitionedWriterBuilder::new( - builder_helper.fanout_partition_writer_builder(delta_builder.clone())?, - writer_param.sink_metrics.iceberg_partition_num.clone(), - ); - let dispatch_builder = - builder_helper.dispatcher_writer_builder(partition_delta_builder, delta_builder)?; - // wrap a layer with collect write metrics - let prometheus_builder = PrometheusWriterBuilder::new( - dispatch_builder, - WriterMetrics::new( - writer_param.sink_metrics.iceberg_write_qps.deref().clone(), - writer_param - .sink_metrics - .iceberg_write_latency - .deref() - .clone(), - ), - ); - let schema = table.current_arrow_schema()?; - let inner_writer = DeltaWriter::new(prometheus_builder.build(&schema).await?); - Ok(Self { - inner_writer: IcebergWriterEnum::Upsert(inner_writer), - schema, - }) + if let Some(extra_partition_col_idx) = writer_param.extra_partition_col_idx { + let partition_delta_builder = builder_helper.precompute_partition_writer_builder( + delta_builder.clone(), + extra_partition_col_idx, + )?; + let dispatch_builder = + builder_helper.dispatcher_writer_builder(partition_delta_builder, delta_builder)?; + // wrap a layer with collect write metrics + let prometheus_builder = PrometheusWriterBuilder::new( + dispatch_builder, + WriterMetrics::new( + writer_param.sink_metrics.iceberg_write_qps.deref().clone(), + writer_param + .sink_metrics + .iceberg_write_latency + .deref() + .clone(), + ), + ); + let schema = Self::schema_with_extra_partition_col(&table, extra_partition_col_idx)?; + let inner_writer = DeltaWriter::new(prometheus_builder.build(&schema).await?); + Ok(Self { + inner_writer: IcebergWriterEnum::Upsert(inner_writer), + schema, + }) + } else { + let partition_delta_builder = + builder_helper.fanout_partition_writer_builder(delta_builder.clone())?; + let dispatch_builder = + builder_helper.dispatcher_writer_builder(partition_delta_builder, delta_builder)?; + // wrap a layer with collect write metrics + let prometheus_builder = PrometheusWriterBuilder::new( + dispatch_builder, + WriterMetrics::new( + writer_param.sink_metrics.iceberg_write_qps.deref().clone(), + writer_param + .sink_metrics + .iceberg_write_latency + .deref() + .clone(), + ), + ); + let schema = table.current_arrow_schema()?; + let inner_writer = DeltaWriter::new(prometheus_builder.build(&schema).await?); + Ok(Self { + inner_writer: IcebergWriterEnum::Upsert(inner_writer), + schema, + }) + } } } @@ -496,13 +761,14 @@ impl SinkWriter for IcebergWriter { let filters = chunk.visibility() & ops.iter().map(|op| *op == Op::Insert).collect::(); chunk.set_visibility(filters); - let chunk = to_record_batch_with_schema(self.schema.clone(), &chunk.compact()) - .map_err(|err| SinkError::Iceberg(anyhow!(err)))?; + let chunk = + to_iceberg_record_batch_with_schema(self.schema.clone(), &chunk.compact()) + .map_err(|err| SinkError::Iceberg(anyhow!(err)))?; writer.write(chunk).await?; } IcebergWriterEnum::Upsert(writer) => { - let chunk = to_record_batch_with_schema(self.schema.clone(), &chunk) + let chunk = to_iceberg_record_batch_with_schema(self.schema.clone(), &chunk) .map_err(|err| SinkError::Iceberg(anyhow!(err)))?; writer @@ -574,19 +840,19 @@ impl WriteResult { fn try_from(value: &SinkMetadata, partition_type: &Any) -> Result { if let Some(Serialized(v)) = &value.metadata { let mut values = if let serde_json::Value::Object(v) = - serde_json::from_slice::(&v.metadata).map_err( - |e| -> SinkError { anyhow!("Can't parse iceberg sink metadata: {}", e).into() }, - )? { + serde_json::from_slice::(&v.metadata) + .context("Can't parse iceberg sink metadata")? + { v } else { - return Err(anyhow!("iceberg sink metadata should be a object").into()); + bail!("iceberg sink metadata should be a object"); }; let data_files: Vec; let delete_files: Vec; if let serde_json::Value::Array(values) = values .remove(DATA_FILES) - .ok_or_else(|| anyhow!("icberg sink metadata should have data_files object"))? + .ok_or_else(|| anyhow!("iceberg sink metadata should have data_files object"))? { data_files = values .into_iter() @@ -594,26 +860,26 @@ impl WriteResult { .collect::, icelake::Error>>() .unwrap(); } else { - return Err(anyhow!("icberg sink metadata should have data_files object").into()); + bail!("iceberg sink metadata should have data_files object"); } if let serde_json::Value::Array(values) = values .remove(DELETE_FILES) - .ok_or_else(|| anyhow!("icberg sink metadata should have data_files object"))? + .ok_or_else(|| anyhow!("iceberg sink metadata should have data_files object"))? { delete_files = values .into_iter() .map(|value| data_file_from_json(value, partition_type.clone())) .collect::, icelake::Error>>() - .map_err(|e| anyhow!("Failed to parse data file from json: {}", e))?; + .context("Failed to parse data file from json")?; } else { - return Err(anyhow!("icberg sink metadata should have data_files object").into()); + bail!("icberg sink metadata should have data_files object"); } Ok(Self { data_files, delete_files, }) } else { - Err(anyhow!("Can't create iceberg sink write result from empty data!").into()) + bail!("Can't create iceberg sink write result from empty data!") } } } @@ -629,7 +895,7 @@ impl<'a> TryFrom<&'a WriteResult> for SinkMetadata { .cloned() .map(data_file_to_json) .collect::, icelake::Error>>() - .map_err(|e| anyhow!("Can't serialize data files to json: {}", e))?, + .context("Can't serialize data files to json")?, ); let json_delete_files = serde_json::Value::Array( value @@ -638,7 +904,7 @@ impl<'a> TryFrom<&'a WriteResult> for SinkMetadata { .cloned() .map(data_file_to_json) .collect::, icelake::Error>>() - .map_err(|e| anyhow!("Can't serialize data files to json: {}", e))?, + .context("Can't serialize data files to json")?, ); let json_value = serde_json::Value::Object( vec![ @@ -650,9 +916,8 @@ impl<'a> TryFrom<&'a WriteResult> for SinkMetadata { ); Ok(SinkMetadata { metadata: Some(Serialized(SerializedMetadata { - metadata: serde_json::to_vec(&json_value).map_err(|e| -> SinkError { - anyhow!("Can't serialized iceberg sink metadata: {}", e).into() - })?, + metadata: serde_json::to_vec(&json_value) + .context("Can't serialize iceberg sink metadata")?, })), }) } @@ -686,23 +951,29 @@ impl SinkCommitCoordinator for IcebergSinkCommitter { txn.append_data_file(s.data_files); txn.append_delete_file(s.delete_files); }); - txn.commit() - .await - .map_err(|err| SinkError::Iceberg(anyhow!(err)))?; + txn.commit().await.map_err(|err| { + tracing::error!(error = %err.as_report(), "Failed to commit iceberg table"); + SinkError::Iceberg(anyhow!(err)) + })?; - tracing::info!("Succeeded to commit ti iceberg table in epoch {epoch}."); + tracing::info!("Succeeded to commit to iceberg table in epoch {epoch}."); Ok(()) } } /// Try to match our schema with iceberg schema. -fn try_matches_arrow_schema(rw_schema: &Schema, arrow_schema: &ArrowSchema) -> Result<()> { +/// `for_source` = true means the schema is used for source, otherwise it's used for sink. +pub fn try_matches_arrow_schema( + rw_schema: &Schema, + arrow_schema: &ArrowSchema, + for_source: bool, +) -> anyhow::Result<()> { if rw_schema.fields.len() != arrow_schema.fields().len() { - return Err(SinkError::Iceberg(anyhow!( - "Schema length not match, ours is {}, and iceberg is {}", + bail!( + "Schema length not match, risingwave is {}, and iceberg is {}", rw_schema.fields.len(), arrow_schema.fields.len() - ))); + ); } let mut schema_fields = HashMap::new(); @@ -713,26 +984,37 @@ fn try_matches_arrow_schema(rw_schema: &Schema, arrow_schema: &ArrowSchema) -> R }); for arrow_field in &arrow_schema.fields { - let our_field_type = schema_fields.get(arrow_field.name()).ok_or_else(|| { - SinkError::Iceberg(anyhow!( - "Field {} not found in our schema", - arrow_field.name() - )) - })?; - - let converted_arrow_data_type = - ArrowDataType::try_from(*our_field_type).map_err(|e| SinkError::Iceberg(anyhow!(e)))?; + let our_field_type = schema_fields + .get(arrow_field.name()) + .ok_or_else(|| anyhow!("Field {} not found in our schema", arrow_field.name()))?; + + // Iceberg source should be able to read iceberg decimal type. + // Since the arrow type default conversion is used by udf, in udf, decimal is converted to + // large binary type which is not compatible with iceberg decimal type, + // so we need to convert it to decimal type manually. + let converted_arrow_data_type = if for_source + && matches!(our_field_type, risingwave_common::types::DataType::Decimal) + { + // RisingWave decimal type cannot specify precision and scale, so we use the default value. + ArrowDataType::Decimal128(38, 0) + } else { + ArrowDataType::try_from(*our_field_type).map_err(|e| anyhow!(e))? + }; let compatible = match (&converted_arrow_data_type, arrow_field.data_type()) { (ArrowDataType::Decimal128(p1, s1), ArrowDataType::Decimal128(p2, s2)) => { - *p1 >= *p2 && *s1 >= *s2 + if for_source { + true + } else { + *p1 >= *p2 && *s1 >= *s2 + } } (left, right) => left == right, }; if !compatible { - return Err(SinkError::Iceberg(anyhow!("Field {}'s type not compatible, ours converted data type {}, iceberg's data type: {}", + bail!("Field {}'s type not compatible, risingwave converted data type {}, iceberg's data type: {}", arrow_field.name(), converted_arrow_data_type, arrow_field.data_type() - ))); + ); } } @@ -741,8 +1023,11 @@ fn try_matches_arrow_schema(rw_schema: &Schema, arrow_schema: &ArrowSchema) -> R #[cfg(test)] mod test { + use std::collections::HashMap; + use risingwave_common::catalog::Field; + use crate::sink::iceberg::IcebergConfig; use crate::source::DataType; #[test] @@ -761,7 +1046,7 @@ mod test { ArrowField::new("c", ArrowDataType::Int32, false), ]); - try_matches_arrow_schema(&risingwave_schema, &arrow_schema).unwrap(); + try_matches_arrow_schema(&risingwave_schema, &arrow_schema, false).unwrap(); let risingwave_schema = Schema::new(vec![ Field::with_name(DataType::Int32, "d"), @@ -775,6 +1060,169 @@ mod test { ArrowField::new("d", ArrowDataType::Int32, false), ArrowField::new("c", ArrowDataType::Int32, false), ]); - try_matches_arrow_schema(&risingwave_schema, &arrow_schema).unwrap(); + try_matches_arrow_schema(&risingwave_schema, &arrow_schema, false).unwrap(); + } + + #[test] + fn test_parse_iceberg_config() { + let values = [ + ("connector", "iceberg"), + ("type", "upsert"), + ("primary_key", "v1"), + ("warehouse.path", "s3://iceberg"), + ("s3.endpoint", "http://127.0.0.1:9301"), + ("s3.access.key", "hummockadmin"), + ("s3.secret.key", "hummockadmin"), + ("s3.region", "us-east-1"), + ("catalog.type", "jdbc"), + ("catalog.name", "demo"), + ("catalog.uri", "jdbc://postgresql://postgres:5432/iceberg"), + ("catalog.jdbc.user", "admin"), + ("catalog.jdbc.password", "123456"), + ("database.name", "demo_db"), + ("table.name", "demo_table"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + let iceberg_config = IcebergConfig::from_hashmap(values).unwrap(); + + let expected_iceberg_config = IcebergConfig { + connector: "iceberg".to_string(), + r#type: "upsert".to_string(), + force_append_only: false, + table_name: "demo_table".to_string(), + database_name: Some("demo_db".to_string()), + catalog_name: Some("demo".to_string()), + catalog_type: Some("jdbc".to_string()), + path: "s3://iceberg".to_string(), + uri: Some("jdbc://postgresql://postgres:5432/iceberg".to_string()), + region: Some("us-east-1".to_string()), + endpoint: Some("http://127.0.0.1:9301".to_string()), + access_key: "hummockadmin".to_string(), + secret_key: "hummockadmin".to_string(), + primary_key: Some(vec!["v1".to_string()]), + java_catalog_props: [("jdbc.user", "admin"), ("jdbc.password", "123456")] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(), + }; + + assert_eq!(iceberg_config, expected_iceberg_config); + + assert_eq!( + &iceberg_config.full_table_name().unwrap().to_string(), + "demo_db.demo_table" + ); + } + + async fn test_create_catalog(configs: HashMap) { + let iceberg_config = IcebergConfig::from_hashmap(configs).unwrap(); + + let table = iceberg_config.load_table().await.unwrap(); + + println!("{:?}", table.table_name()); + } + + #[tokio::test] + #[ignore] + async fn test_storage_catalog() { + let values = [ + ("connector", "iceberg"), + ("type", "append-only"), + ("force_append_only", "true"), + ("s3.endpoint", "http://127.0.0.1:9301"), + ("s3.access.key", "hummockadmin"), + ("s3.secret.key", "hummockadmin"), + ("s3.region", "us-east-1"), + ("catalog.name", "demo"), + ("catalog.type", "storage"), + ("warehouse.path", "s3://icebergdata/demo"), + ("database.name", "s1"), + ("table.name", "t1"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + test_create_catalog(values).await; + } + + #[tokio::test] + #[ignore] + async fn test_rest_catalog() { + let values = [ + ("connector", "iceberg"), + ("type", "append-only"), + ("force_append_only", "true"), + ("s3.endpoint", "http://127.0.0.1:9301"), + ("s3.access.key", "hummockadmin"), + ("s3.secret.key", "hummockadmin"), + ("s3.region", "us-east-1"), + ("catalog.name", "demo"), + ("catalog.type", "rest"), + ("catalog.uri", "http://192.168.167.4:8181"), + ("warehouse.path", "s3://icebergdata/demo"), + ("database.name", "s1"), + ("table.name", "t1"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + test_create_catalog(values).await; + } + + #[tokio::test] + #[ignore] + async fn test_jdbc_catalog() { + let values = [ + ("connector", "iceberg"), + ("type", "append-only"), + ("force_append_only", "true"), + ("s3.endpoint", "http://127.0.0.1:9301"), + ("s3.access.key", "hummockadmin"), + ("s3.secret.key", "hummockadmin"), + ("s3.region", "us-east-1"), + ("catalog.name", "demo"), + ("catalog.type", "jdbc"), + ("catalog.uri", "jdbc:postgresql://localhost:5432/iceberg"), + ("catalog.jdbc.user", "admin"), + ("catalog.jdbc.password", "123456"), + ("warehouse.path", "s3://icebergdata/demo"), + ("database.name", "s1"), + ("table.name", "t1"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + test_create_catalog(values).await; + } + + #[tokio::test] + #[ignore] + async fn test_hive_catalog() { + let values = [ + ("connector", "iceberg"), + ("type", "append-only"), + ("force_append_only", "true"), + ("s3.endpoint", "http://127.0.0.1:9301"), + ("s3.access.key", "hummockadmin"), + ("s3.secret.key", "hummockadmin"), + ("s3.region", "us-east-1"), + ("catalog.name", "demo"), + ("catalog.type", "hive"), + ("catalog.uri", "thrift://localhost:9083"), + ("warehouse.path", "s3://icebergdata/demo"), + ("database.name", "s1"), + ("table.name", "t1"), + ] + .into_iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + + test_create_catalog(values).await; } } diff --git a/src/connector/src/sink/kafka.rs b/src/connector/src/sink/kafka.rs index 0c59780916cb4..6d5407578b29d 100644 --- a/src/connector/src/sink/kafka.rs +++ b/src/connector/src/sink/kafka.rs @@ -29,11 +29,12 @@ use risingwave_common::catalog::Schema; use serde_derive::Deserialize; use serde_with::{serde_as, DisplayFromStr}; use strum_macros::{Display, EnumString}; +use thiserror_ext::AsReport; use with_options::WithOptions; use super::catalog::{SinkFormat, SinkFormatDesc}; use super::{Sink, SinkError, SinkParam}; -use crate::common::{KafkaCommon, RdKafkaPropertiesCommon}; +use crate::common::{KafkaCommon, KafkaPrivateLinkCommon, RdKafkaPropertiesCommon}; use crate::sink::catalog::desc::SinkDesc; use crate::sink::formatter::SinkFormatterImpl; use crate::sink::log_store::DeliveryFutureManagerAddFuture; @@ -232,6 +233,9 @@ pub struct KafkaConfig { #[serde(flatten)] pub rdkafka_properties_producer: RdKafkaPropertiesProducer, + + #[serde(flatten)] + pub privatelink_common: KafkaPrivateLinkCommon, } impl KafkaConfig { @@ -261,6 +265,7 @@ impl From for KafkaProperties { common: val.common, rdkafka_properties_common: val.rdkafka_properties_common, rdkafka_properties_consumer: Default::default(), + privatelink_common: val.privatelink_common, unknown_fields: Default::default(), } } @@ -403,7 +408,7 @@ impl KafkaSinkWriter { // Create the producer context, will be used to create the producer let producer_ctx = PrivateLinkProducerContext::new( - config.common.broker_rewrite_map.clone(), + config.privatelink_common.broker_rewrite_map.clone(), // fixme: enable kafka native metrics for sink None, None, @@ -474,10 +479,10 @@ impl<'w> KafkaPayloadWriter<'w> { // We can retry for another round after sleeping for sometime Err((e, rec)) => { tracing::warn!( - "producing message (key {:?}) to topic {} failed, err {:?}.", + error = %e.as_report(), + "producing message (key {:?}) to topic {} failed", rec.key.map(|k| k.to_bytes()), rec.topic, - e ); record = rec; match e { @@ -656,6 +661,8 @@ mod test { "properties.sasl.password".to_string() => "test".to_string(), "properties.retry.max".to_string() => "20".to_string(), "properties.retry.interval".to_string() => "500ms".to_string(), + // PrivateLink + "broker.rewrite.endpoints".to_string() => "{\"broker1\": \"10.0.0.1:8001\"}".to_string(), }; let config = KafkaConfig::from_hashmap(properties).unwrap(); assert_eq!(config.common.brokers, "localhost:9092"); @@ -663,6 +670,12 @@ mod test { assert_eq!(config.max_retry_num, 20); assert_eq!(config.retry_interval, Duration::from_millis(500)); + // PrivateLink fields + let hashmap: HashMap = hashmap! { + "broker1".to_string() => "10.0.0.1:8001".to_string() + }; + assert_eq!(config.privatelink_common.broker_rewrite_map, Some(hashmap)); + // Optional fields eliminated. let properties: HashMap = hashmap! { // "connector".to_string() => "kafka".to_string(), diff --git a/src/connector/src/sink/kinesis.rs b/src/connector/src/sink/kinesis.rs index aca370d7ab1b3..04127a373dd28 100644 --- a/src/connector/src/sink/kinesis.rs +++ b/src/connector/src/sink/kinesis.rs @@ -14,8 +14,7 @@ use std::collections::HashMap; -use anyhow::anyhow; -use aws_sdk_kinesis::error::DisplayErrorContext; +use anyhow::{anyhow, Context}; use aws_sdk_kinesis::operation::put_record::PutRecordOutput; use aws_sdk_kinesis::primitives::Blob; use aws_sdk_kinesis::Client as KinesisClient; @@ -106,10 +105,8 @@ impl Sink for KinesisSink { .stream_name(&self.config.common.stream_name) .send() .await - .map_err(|e| { - tracing::warn!("failed to list shards: {}", DisplayErrorContext(&e)); - SinkError::Kinesis(anyhow!("failed to list shards: {}", DisplayErrorContext(e))) - })?; + .context("failed to list shards") + .map_err(SinkError::Kinesis)?; Ok(()) } @@ -176,7 +173,7 @@ impl KinesisSinkWriter { .common .build_client() .await - .map_err(SinkError::Kinesis)?; + .map_err(|err| SinkError::Kinesis(anyhow!(err)))?; Ok(Self { config: config.clone(), formatter, @@ -201,18 +198,8 @@ impl KinesisSinkPayloadWriter { }, ) .await - .map_err(|e| { - tracing::warn!( - "failed to put record: {} to {}", - DisplayErrorContext(&e), - self.config.common.stream_name - ); - SinkError::Kinesis(anyhow!( - "failed to put record: {} to {}", - DisplayErrorContext(e), - self.config.common.stream_name - )) - }) + .with_context(|| format!("failed to put record to {}", self.config.common.stream_name)) + .map_err(SinkError::Kinesis) } } diff --git a/src/connector/src/sink/log_store.rs b/src/connector/src/sink/log_store.rs index 3547495f483d9..c2cdf7756c598 100644 --- a/src/connector/src/sink/log_store.rs +++ b/src/connector/src/sink/log_store.rs @@ -19,9 +19,9 @@ use std::future::{poll_fn, Future}; use std::sync::Arc; use std::task::Poll; -use anyhow::anyhow; use futures::{TryFuture, TryFutureExt}; use risingwave_common::array::StreamChunk; +use risingwave_common::bail; use risingwave_common::buffer::Bitmap; use risingwave_common::util::epoch::{EpochPair, INVALID_EPOCH}; @@ -62,13 +62,13 @@ impl TruncateOffset { } } - pub fn check_next_offset(&self, next_offset: TruncateOffset) -> anyhow::Result<()> { + pub fn check_next_offset(&self, next_offset: TruncateOffset) -> LogStoreResult<()> { if *self >= next_offset { - Err(anyhow!( + bail!( "next offset {:?} should be later than current offset {:?}", next_offset, self - )) + ) } else { Ok(()) } @@ -81,22 +81,22 @@ impl TruncateOffset { .. } => { if epoch != *offset_epoch { - return Err(anyhow!( + bail!( "new item epoch {} not match current chunk offset epoch {}", epoch, offset_epoch - )); + ); } } TruncateOffset::Barrier { epoch: offset_epoch, } => { if epoch <= *offset_epoch { - return Err(anyhow!( + bail!( "new item epoch {} not exceed barrier offset epoch {}", epoch, offset_epoch - )); + ); } } } @@ -535,6 +535,7 @@ mod tests { use tokio::sync::oneshot; use tokio::sync::oneshot::Receiver; + use super::LogStoreResult; use crate::sink::log_store::{DeliveryFutureManager, TruncateOffset}; #[test] @@ -588,7 +589,7 @@ mod tests { } type TestFuture = impl TryFuture + Unpin + 'static; - fn to_test_future(rx: Receiver>) -> TestFuture { + fn to_test_future(rx: Receiver>) -> TestFuture { async move { rx.await.unwrap() }.boxed() } diff --git a/src/connector/src/sink/mock_coordination_client.rs b/src/connector/src/sink/mock_coordination_client.rs new file mode 100644 index 0000000000000..8089a99e32ec0 --- /dev/null +++ b/src/connector/src/sink/mock_coordination_client.rs @@ -0,0 +1,170 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_common::buffer::Bitmap; +use risingwave_pb::connector_service::coordinate_response::{ + self, CommitResponse, StartCoordinationResponse, +}; +use risingwave_pb::connector_service::{ + coordinate_request, CoordinateRequest, CoordinateResponse, PbSinkParam, +}; +use risingwave_rpc_client::error::RpcError; +use risingwave_rpc_client::{CoordinatorStreamHandle, SinkCoordinationRpcClient}; +use tokio::sync::mpsc::{self, Receiver}; +use tokio_stream::wrappers::ReceiverStream; +use tonic::Status; + +use super::boxed::BoxCoordinator; +use super::{SinkParam, BOUNDED_CHANNEL_SIZE}; + +#[derive(Clone)] +pub enum SinkCoordinationRpcClientEnum { + SinkCoordinationRpcClient(SinkCoordinationRpcClient), + MockSinkCoordinationRpcClient(MockSinkCoordinationRpcClient), +} + +impl SinkCoordinationRpcClientEnum { + pub async fn new_stream_handle( + self, + param: SinkParam, + vnode_bitmap: Bitmap, + ) -> super::Result { + match self { + SinkCoordinationRpcClientEnum::SinkCoordinationRpcClient( + sink_coordination_rpc_client, + ) => Ok(CoordinatorStreamHandle::new( + sink_coordination_rpc_client, + param.to_proto(), + vnode_bitmap, + ) + .await?), + SinkCoordinationRpcClientEnum::MockSinkCoordinationRpcClient( + mock_sink_coordination_rpc_client, + ) => Ok(mock_sink_coordination_rpc_client + .new_stream_handle(param.to_proto(), vnode_bitmap) + .await?), + } + } +} + +#[derive(Clone)] +pub struct MockMetaClient { + mock_coordinator_committer: std::sync::Arc>, +} +impl MockMetaClient { + pub fn new(mock_coordinator_committer: BoxCoordinator) -> Self { + Self { + mock_coordinator_committer: std::sync::Arc::new(tokio::sync::Mutex::new( + mock_coordinator_committer, + )), + } + } + + pub fn sink_coordinate_client(&self) -> MockSinkCoordinationRpcClient { + MockSinkCoordinationRpcClient::new(self.mock_coordinator_committer.clone()) + } +} + +#[derive(Clone)] +pub struct MockSinkCoordinationRpcClient { + mock_coordinator_committer: std::sync::Arc>, +} + +impl MockSinkCoordinationRpcClient { + pub fn new( + mock_coordinator_committer: std::sync::Arc>, + ) -> Self { + Self { + mock_coordinator_committer, + } + } + + pub async fn new_stream_handle( + &self, + param: PbSinkParam, + vnode_bitmap: Bitmap, + ) -> std::result::Result { + CoordinatorStreamHandle::new_with_init_stream(param, vnode_bitmap, |rx| async move { + self.coordinate(rx).await + }) + .await + } + + pub async fn coordinate( + &self, + mut receiver_stream: Receiver, + ) -> std::result::Result< + tonic::Response>>, + Status, + > { + match receiver_stream.try_recv() { + Ok(CoordinateRequest { + msg: + Some(risingwave_pb::connector_service::coordinate_request::Msg::StartRequest( + coordinate_request::StartCoordinationRequest { + param: Some(_param), + vnode_bitmap: Some(_vnode_bitmap), + }, + )), + }) => (), + msg => { + return Err(Status::invalid_argument(format!( + "expected CoordinateRequest::StartRequest in the first request, get {:?}", + msg + ))); + } + }; + + let (response_tx, response_rx) = + mpsc::channel::>(BOUNDED_CHANNEL_SIZE); + let response_tx = std::sync::Arc::new(response_tx); + response_tx + .send(Ok(CoordinateResponse { + msg: Some(coordinate_response::Msg::StartResponse( + StartCoordinationResponse {}, + )), + })) + .await + .map_err(|e| Status::from_error(Box::new(e)))?; + + let mock_coordinator_committer = self.mock_coordinator_committer.clone(); + let response_tx_clone = response_tx.clone(); + tokio::spawn(async move { + loop { + match receiver_stream.recv().await { + Some(CoordinateRequest { + msg: + Some(risingwave_pb::connector_service::coordinate_request::Msg::CommitRequest(coordinate_request::CommitRequest { + epoch, + metadata, + })), + }) => { + mock_coordinator_committer.clone().lock().await.commit(epoch, vec![metadata.unwrap()]).await.map_err(|e| Status::from_error(Box::new(e)))?; + response_tx_clone.clone().send(Ok(CoordinateResponse { + msg: Some(coordinate_response::Msg::CommitResponse(CommitResponse{epoch})), + })).await.map_err(|e| Status::from_error(Box::new(e)))?; + }, + msg => { + return Err::, tonic::Status>(Status::invalid_argument(format!( + "expected CoordinateRequest::CommitRequest , get {:?}", + msg + ))); + } + } + } + }); + + Ok(tonic::Response::new(ReceiverStream::new(response_rx))) + } +} diff --git a/src/connector/src/sink/mod.rs b/src/connector/src/sink/mod.rs index a9baaad178ea4..6abe8d93b5956 100644 --- a/src/connector/src/sink/mod.rs +++ b/src/connector/src/sink/mod.rs @@ -27,6 +27,7 @@ pub mod iceberg; pub mod kafka; pub mod kinesis; pub mod log_store; +pub mod mock_coordination_client; pub mod nats; pub mod pulsar; pub mod redis; @@ -47,7 +48,6 @@ use anyhow::anyhow; use async_trait::async_trait; use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::{ColumnDesc, Field, Schema}; -use risingwave_common::error::{anyhow_error, ErrorCode, RwError}; use risingwave_common::metrics::{ LabelGuardedHistogram, LabelGuardedIntCounter, LabelGuardedIntGauge, }; @@ -56,15 +56,19 @@ use risingwave_pb::connector_service::{PbSinkParam, SinkMetadata, TableSchema}; use risingwave_rpc_client::error::RpcError; use risingwave_rpc_client::MetaClient; use thiserror::Error; +use thiserror_ext::AsReport; pub use tracing; use self::catalog::{SinkFormatDesc, SinkType}; +use self::mock_coordination_client::{MockMetaClient, SinkCoordinationRpcClientEnum}; +use crate::error::ConnectorError; use crate::sink::catalog::desc::SinkDesc; use crate::sink::catalog::{SinkCatalog, SinkId}; use crate::sink::log_store::{LogReader, LogStoreReadItem, LogStoreResult, TruncateOffset}; use crate::sink::writer::SinkWriter; use crate::ConnectorParams; +const BOUNDED_CHANNEL_SIZE: usize = 16; #[macro_export] macro_rules! for_all_sinks { ($macro:path $(, $arg:tt)*) => { @@ -265,8 +269,36 @@ pub struct SinkWriterParam { pub connector_params: ConnectorParams, pub executor_id: u64, pub vnode_bitmap: Option, - pub meta_client: Option, + pub meta_client: Option, pub sink_metrics: SinkMetrics, + // The val has two effect: + // 1. Indicates that the sink will accpect the data chunk with extra partition value column. + // 2. The index of the extra partition value column. + // More detail of partition value column, see `PartitionComputeInfo` + pub extra_partition_col_idx: Option, +} + +#[derive(Clone)] +pub enum SinkMetaClient { + MetaClient(MetaClient), + MockMetaClient(MockMetaClient), +} + +impl SinkMetaClient { + pub async fn sink_coordinate_client(&self) -> SinkCoordinationRpcClientEnum { + match self { + SinkMetaClient::MetaClient(meta_client) => { + SinkCoordinationRpcClientEnum::SinkCoordinationRpcClient( + meta_client.sink_coordinate_client().await, + ) + } + SinkMetaClient::MockMetaClient(mock_meta_client) => { + SinkCoordinationRpcClientEnum::MockSinkCoordinationRpcClient( + mock_meta_client.sink_coordinate_client(), + ) + } + } + } } impl SinkWriterParam { @@ -277,6 +309,7 @@ impl SinkWriterParam { vnode_bitmap: Default::default(), meta_client: Default::default(), sink_metrics: SinkMetrics::for_test(), + extra_partition_col_idx: Default::default(), } } } @@ -500,40 +533,40 @@ pub enum SinkError { #[backtrace] anyhow::Error, ), + #[error(transparent)] + Connector( + #[from] + #[backtrace] + ConnectorError, + ), } impl From for SinkError { fn from(value: icelake::Error) -> Self { - SinkError::Iceberg(anyhow_error!("{}", value)) + SinkError::Iceberg(anyhow!(value)) } } impl From for SinkError { fn from(value: RpcError) -> Self { - SinkError::Remote(anyhow_error!("{}", value)) + SinkError::Remote(anyhow!(value)) } } impl From for SinkError { fn from(value: ClickHouseError) -> Self { - SinkError::ClickHouse(format!("{}", value)) + SinkError::ClickHouse(value.to_report_string()) } } impl From for SinkError { fn from(value: DeltaTableError) -> Self { - SinkError::DeltaLake(anyhow_error!("{}", value)) + SinkError::DeltaLake(anyhow!(value)) } } impl From for SinkError { fn from(value: RedisError) -> Self { - SinkError::Redis(format!("{}", value)) - } -} - -impl From for RwError { - fn from(e: SinkError) -> Self { - ErrorCode::SinkError(Box::new(e)).into() + SinkError::Redis(value.to_report_string()) } } diff --git a/src/connector/src/sink/nats.rs b/src/connector/src/sink/nats.rs index 01bac5e6b048c..fc6afc379eb76 100644 --- a/src/connector/src/sink/nats.rs +++ b/src/connector/src/sink/nats.rs @@ -14,11 +14,10 @@ use core::fmt::Debug; use std::collections::HashMap; -use anyhow::anyhow; +use anyhow::{anyhow, Context as _}; use async_nats::jetstream::context::Context; use risingwave_common::array::StreamChunk; use risingwave_common::catalog::Schema; -use risingwave_common::error::anyhow_error; use serde_derive::Deserialize; use serde_with::serde_as; use tokio_retry::strategy::{jitter, ExponentialBackoff}; @@ -108,15 +107,9 @@ impl Sink for NatsSink { "Nats sink only support append-only mode" ))); } - match self.config.common.build_client().await { - Ok(_client) => {} - Err(error) => { - return Err(SinkError::Nats(anyhow_error!( - "validate nats sink error: {:?}", - error - ))); - } - } + let _client = (self.config.common.build_client().await) + .context("validate nats sink error") + .map_err(SinkError::Nats)?; Ok(()) } @@ -135,7 +128,7 @@ impl NatsSinkWriter { .common .build_context() .await - .map_err(|e| SinkError::Nats(anyhow_error!("nats sink error: {:?}", e)))?; + .map_err(|e| SinkError::Nats(anyhow!(e)))?; Ok::<_, SinkError>(Self { config: config.clone(), context, @@ -160,13 +153,15 @@ impl NatsSinkWriter { self.context .publish(self.config.common.subject.clone(), item.into()) .await - .map_err(|e| SinkError::Nats(anyhow_error!("nats sink error: {:?}", e)))?; + .context("nats sink error") + .map_err(SinkError::Nats)?; } Ok::<_, SinkError>(()) }, ) .await - .map_err(|e| SinkError::Nats(anyhow_error!("nats sink error: {:?}", e))) + .context("nats sink error") + .map_err(SinkError::Nats) } } diff --git a/src/connector/src/sink/redis.rs b/src/connector/src/sink/redis.rs index 344201981fd5c..f1a07b66cb692 100644 --- a/src/connector/src/sink/redis.rs +++ b/src/connector/src/sink/redis.rs @@ -30,6 +30,7 @@ use super::formatter::SinkFormatterImpl; use super::writer::FormattedSink; use super::{SinkError, SinkParam}; use crate::dispatch_sink_formatter_str_key_impl; +use crate::error::ConnectorResult; use crate::sink::log_store::DeliveryFutureManagerAddFuture; use crate::sink::writer::{ AsyncTruncateLogSinkerOf, AsyncTruncateSinkWriter, AsyncTruncateSinkWriterExt, @@ -47,7 +48,7 @@ pub struct RedisCommon { } impl RedisCommon { - pub(crate) fn build_client(&self) -> anyhow::Result { + pub(crate) fn build_client(&self) -> ConnectorResult { let client = RedisClient::open(self.url.clone())?; Ok(client) } @@ -63,7 +64,7 @@ impl RedisConfig { pub fn from_hashmap(properties: HashMap) -> Result { let config = serde_json::from_value::(serde_json::to_value(properties).unwrap()) - .map_err(|e| SinkError::Config(anyhow!("{:?}", e)))?; + .map_err(|e| SinkError::Config(anyhow!(e)))?; Ok(config) } } diff --git a/src/connector/src/sink/remote.rs b/src/connector/src/sink/remote.rs index 320f77c6a47ba..4c4a662f83178 100644 --- a/src/connector/src/sink/remote.rs +++ b/src/connector/src/sink/remote.rs @@ -26,10 +26,9 @@ use itertools::Itertools; use jni::JavaVM; use prost::Message; use risingwave_common::array::StreamChunk; +use risingwave_common::bail; use risingwave_common::catalog::{ColumnDesc, ColumnId}; -use risingwave_common::error::anyhow_error; use risingwave_common::types::DataType; -use risingwave_common::util::drop_either_future; use risingwave_jni_core::jvm_runtime::JVM; use risingwave_jni_core::{ call_static_method, gen_class_name, JniReceiverType, JniSenderType, JniSinkWriterStreamRequest, @@ -49,6 +48,8 @@ use risingwave_rpc_client::{ BidiStreamReceiver, BidiStreamSender, SinkCoordinatorStreamHandle, SinkWriterStreamHandle, DEFAULT_BUFFER_SIZE, }; +use rw_futures_util::drop_either_future; +use thiserror_ext::AsReport; use tokio::sync::mpsc; use tokio::sync::mpsc::{unbounded_channel, Receiver, Sender}; use tokio::task::spawn_blocking; @@ -56,9 +57,10 @@ use tokio_stream::wrappers::ReceiverStream; use tracing::warn; use super::elasticsearch::{StreamChunkConverter, ES_OPTION_DELIMITER}; +use crate::error::ConnectorResult; use crate::sink::catalog::desc::SinkDesc; use crate::sink::coordinate::CoordinatedSinkWriter; -use crate::sink::log_store::{LogStoreReadItem, TruncateOffset}; +use crate::sink::log_store::{LogStoreReadItem, LogStoreResult, TruncateOffset}; use crate::sink::writer::{LogSinkerOf, SinkWriter, SinkWriterExt}; use crate::sink::{ DummySinkCommitCoordinator, LogSinker, Result, Sink, SinkCommitCoordinator, SinkError, @@ -157,14 +159,12 @@ impl Sink for RemoteSink { } } -async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> anyhow::Result<()> { +async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> ConnectorResult<()> { if sink_name == ElasticSearchSink::SINK_NAME && param.downstream_pk.len() > 1 && param.properties.get(ES_OPTION_DELIMITER).is_none() { - return Err(anyhow_error!( - "Es sink only support single pk or pk with delimiter option" - )); + bail!("Es sink only support single pk or pk with delimiter option"); } // FIXME: support struct and array in stream sink param.columns.iter().map(|col| { @@ -188,7 +188,7 @@ async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> anyhow::Res if (sink_name==ElasticSearchSink::SINK_NAME) | matches!(list.as_ref(), DataType::Int16 | DataType::Int32 | DataType::Int64 | DataType::Float32 | DataType::Float64 | DataType::Varchar){ Ok(()) } else{ - Err(SinkError::Remote(anyhow_error!( + Err(SinkError::Remote(anyhow!( "Remote sink only support list, got {:?}: {:?}", col.name, col.data_type, @@ -199,14 +199,14 @@ async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> anyhow::Res if sink_name==ElasticSearchSink::SINK_NAME{ Ok(()) }else{ - Err(SinkError::Remote(anyhow_error!( + Err(SinkError::Remote(anyhow!( "Only Es sink support struct, got {:?}: {:?}", col.name, col.data_type, ))) } }, - DataType::Serial | DataType::Int256 => Err(SinkError::Remote(anyhow_error!( + DataType::Serial | DataType::Int256 => Err(SinkError::Remote(anyhow!( "remote sink supports Int16, Int32, Int64, Float32, Float64, Boolean, Decimal, Time, Date, Interval, Jsonb, Timestamp, Timestamptz, Bytea, List and Varchar, (Es sink support Struct) got {:?}: {:?}", col.name, col.data_type, @@ -215,7 +215,7 @@ async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> anyhow::Res let jvm = JVM.get_or_init()?; let sink_param = param.to_proto(); - spawn_blocking(move || { + spawn_blocking(move || -> anyhow::Result<()> { let mut env = jvm.attach_current_thread()?; let validate_sink_request = ValidateSinkRequest { sink_param: Some(sink_param), @@ -236,16 +236,13 @@ async fn validate_remote_sink(param: &SinkParam, sink_name: &str) -> anyhow::Res validate_sink_response.error.map_or_else( || Ok(()), // If there is no error message, return Ok here. - |err| { - Err(anyhow!(format!( - "sink cannot pass validation: {}", - err.error_message - ))) - }, + |err| bail!("sink cannot pass validation: {}", err.error_message), ) }) .await - .context("JoinHandle returns error")? + .context("JoinHandle returns error")??; + + Ok(()) } pub struct RemoteLogSinker { @@ -338,12 +335,11 @@ impl LogSinker for RemoteLogSinker { anyhow!("get unsent offset {:?} in response", persisted_offset) })?; if sent_offset != persisted_offset { - return Err(anyhow!( + bail!( "new response offset {:?} not match the buffer offset {:?}", persisted_offset, sent_offset - ) - .into()); + ); } if let (TruncateOffset::Barrier { .. }, Some(start_time)) = @@ -366,13 +362,13 @@ impl LogSinker for RemoteLogSinker { loop { let either_result: futures::future::Either< Option, - anyhow::Result<(u64, LogStoreReadItem)>, + LogStoreResult<(u64, LogStoreReadItem)>, > = drop_either_future( select(pin!(response_rx.recv()), pin!(log_reader.next_item())).await, ); match either_result { futures::future::Either::Left(opt) => { - let response = opt.ok_or_else(|| anyhow!("end of response stream"))?; + let response = opt.context("end of response stream")?; match response { SinkWriterStreamResponse { response: @@ -516,7 +512,7 @@ impl Sink for CoordinatedRemoteSink { .await, self.param.clone(), writer_param.vnode_bitmap.ok_or_else(|| { - SinkError::Remote(anyhow_error!( + SinkError::Remote(anyhow!( "sink needs coordination should not have singleton input" )) })?, @@ -569,7 +565,7 @@ impl CoordinatedRemoteSinkWriter { } fn for_test( - response_receiver: Receiver>, + response_receiver: Receiver>, request_sender: Sender, ) -> CoordinatedRemoteSinkWriter { let properties = HashMap::from([("output.path".to_string(), "/tmp/rw".to_string())]); @@ -602,9 +598,7 @@ impl SinkWriter for CoordinatedRemoteSinkWriter { .inc_by(cardinality as _); let epoch = self.epoch.ok_or_else(|| { - SinkError::Remote(anyhow_error!( - "epoch has not been initialize, call `begin_epoch`" - )) + SinkError::Remote(anyhow!("epoch has not been initialize, call `begin_epoch`")) })?; let batch_id = self.batch_id; self.stream_handle @@ -626,16 +620,14 @@ impl SinkWriter for CoordinatedRemoteSinkWriter { async fn barrier(&mut self, is_checkpoint: bool) -> Result> { let epoch = self.epoch.ok_or_else(|| { - SinkError::Remote(anyhow_error!( - "epoch has not been initialize, call `begin_epoch`" - )) + SinkError::Remote(anyhow!("epoch has not been initialize, call `begin_epoch`")) })?; if is_checkpoint { // TODO: add metrics to measure commit time let rsp = self.stream_handle.commit(epoch).await?; rsp.metadata .ok_or_else(|| { - SinkError::Remote(anyhow_error!( + SinkError::Remote(anyhow!( "get none metadata in commit response for coordinated sink writer" )) }) @@ -771,11 +763,13 @@ impl EmbeddedConnectorClient { let jvm = self.jvm; std::thread::spawn(move || { - let mut env = match jvm.attach_current_thread() { + let mut env = match jvm + .attach_current_thread() + .context("failed to attach current thread") + { Ok(env) => env, Err(e) => { - let _ = response_tx - .blocking_send(Err(anyhow!("failed to attach current thread: {:?}", e))); + let _ = response_tx.blocking_send(Err(e)); return; } }; @@ -794,7 +788,7 @@ impl EmbeddedConnectorClient { tracing::info!("end of jni call {}::{}", class_name, method_name); } Err(e) => { - tracing::error!("jni call error: {:?}", e); + tracing::error!(error = %e.as_report(), "jni call error"); } }; }); diff --git a/src/connector/src/sink/starrocks.rs b/src/connector/src/sink/starrocks.rs index 34286a3b6e991..edefaf7aa1201 100644 --- a/src/connector/src/sink/starrocks.rs +++ b/src/connector/src/sink/starrocks.rs @@ -29,6 +29,8 @@ use serde::Deserialize; use serde_derive::Serialize; use serde_json::Value; use serde_with::serde_as; +use thiserror_ext::AsReport; +use with_options::WithOptions; use super::doris_starrocks_connector::{ HeaderBuilder, InserterInner, InserterInnerBuilder, DORIS_SUCCESS_STATUS, STARROCKS_DELETE_SIGN, @@ -44,26 +46,35 @@ const STARROCK_MYSQL_PREFER_SOCKET: &str = "false"; const STARROCK_MYSQL_MAX_ALLOWED_PACKET: usize = 1024; const STARROCK_MYSQL_WAIT_TIMEOUT: usize = 28800; -#[derive(Deserialize, Debug, Clone)] +#[derive(Deserialize, Debug, Clone, WithOptions)] pub struct StarrocksCommon { + /// The StarRocks host address. #[serde(rename = "starrocks.host")] pub host: String, - #[serde(rename = "starrocks.mysqlport")] + /// The port to the MySQL server of StarRocks FE. + #[serde(rename = "starrocks.mysqlport", alias = "starrocks.query_port")] pub mysql_port: String, - #[serde(rename = "starrocks.httpport")] + /// The port to the HTTP server of StarRocks FE. + #[serde(rename = "starrocks.httpport", alias = "starrocks.http_port")] pub http_port: String, + /// The user name used to access the StarRocks database. #[serde(rename = "starrocks.user")] pub user: String, + /// The password associated with the user. #[serde(rename = "starrocks.password")] pub password: String, + /// The StarRocks database where the target table is located #[serde(rename = "starrocks.database")] pub database: String, + /// The StarRocks table you want to sink data to. #[serde(rename = "starrocks.table")] pub table: String, + #[serde(rename = "starrocks.partial_update")] + pub partial_update: Option, } #[serde_as] -#[derive(Clone, Debug, Deserialize)] +#[derive(Clone, Debug, Deserialize, WithOptions)] pub struct StarrocksConfig { #[serde(flatten)] pub common: StarrocksCommon, @@ -117,8 +128,8 @@ impl StarrocksSink { starrocks_columns_desc: HashMap, ) -> Result<()> { let rw_fields_name = self.schema.fields(); - if rw_fields_name.len().ne(&starrocks_columns_desc.len()) { - return Err(SinkError::Starrocks("The length of the RisingWave column must be equal to the length of the starrocks column".to_string())); + if rw_fields_name.len() > starrocks_columns_desc.len() { + return Err(SinkError::Starrocks("The length of the RisingWave column must be equal or less to the length of the starrocks column".to_string())); } for i in rw_fields_name { @@ -128,7 +139,7 @@ impl StarrocksSink { i.name )) })?; - if !Self::check_and_correct_column_type(&i.data_type, value.to_string())? { + if !Self::check_and_correct_column_type(&i.data_type, value)? { return Err(SinkError::Starrocks(format!( "Column type don't match, column name is {:?}. starrocks type is {:?} risingwave type is {:?} ",i.name,value,i.data_type ))); @@ -139,7 +150,7 @@ impl StarrocksSink { fn check_and_correct_column_type( rw_data_type: &DataType, - starrocks_data_type: String, + starrocks_data_type: &String, ) -> Result { match rw_data_type { risingwave_common::types::DataType::Boolean => { @@ -164,35 +175,37 @@ impl StarrocksSink { Ok(starrocks_data_type.contains("varchar")) } risingwave_common::types::DataType::Time => Err(SinkError::Starrocks( - "starrocks can not support Time".to_string(), + "TIME is not supported for Starrocks sink. Please convert to VARCHAR or other supported types.".to_string(), )), risingwave_common::types::DataType::Timestamp => { Ok(starrocks_data_type.contains("datetime")) } risingwave_common::types::DataType::Timestamptz => Err(SinkError::Starrocks( - "starrocks can not support Timestamptz".to_string(), + "TIMESTAMP WITH TIMEZONE is not supported for Starrocks sink as Starrocks doesn't store time values with timezone information. Please convert to TIMESTAMP first.".to_string(), )), risingwave_common::types::DataType::Interval => Err(SinkError::Starrocks( - "starrocks can not support Interval".to_string(), + "INTERVAL is not supported for Starrocks sink. Please convert to VARCHAR or other supported types.".to_string(), )), - // todo! Validate the type struct and list risingwave_common::types::DataType::Struct(_) => Err(SinkError::Starrocks( - "starrocks can not support import struct".to_string(), + "STRUCT is not supported for Starrocks sink.".to_string(), )), - risingwave_common::types::DataType::List(_) => { - Ok(starrocks_data_type.contains("unknown")) + risingwave_common::types::DataType::List(list) => { + // For compatibility with older versions starrocks + if starrocks_data_type.contains("unknown") { + return Ok(true); + } + let check_result = Self::check_and_correct_column_type(list.as_ref(), starrocks_data_type)?; + Ok(check_result && starrocks_data_type.contains("array")) } risingwave_common::types::DataType::Bytea => Err(SinkError::Starrocks( - "starrocks can not support Bytea".to_string(), - )), - risingwave_common::types::DataType::Jsonb => Err(SinkError::Starrocks( - "starrocks can not support import json".to_string(), + "BYTEA is not supported for Starrocks sink. Please convert to VARCHAR or other supported types.".to_string(), )), + risingwave_common::types::DataType::Jsonb => Ok(starrocks_data_type.contains("json")), risingwave_common::types::DataType::Serial => { Ok(starrocks_data_type.contains("bigint")) } risingwave_common::types::DataType::Int256 => Err(SinkError::Starrocks( - "starrocks can not support Int256".to_string(), + "INT256 is not supported for Starrocks sink.".to_string(), )), } } @@ -314,28 +327,32 @@ impl StarrocksSinkWriter { .first() .ok_or_else(|| SinkError::Starrocks("must have next".to_string()))? .parse::() - .map_err(|e| SinkError::Starrocks(format!("starrocks sink error {}", e)))?; + .map_err(|e| { + SinkError::Starrocks(format!("starrocks sink error: {}", e.as_report())) + })?; let scale = decimal_all .last() .ok_or_else(|| SinkError::Starrocks("must have next".to_string()))? .parse::() - .map_err(|e| SinkError::Starrocks(format!("starrocks sink error {}", e)))?; + .map_err(|e| { + SinkError::Starrocks(format!("starrocks sink error: {}", e.as_report())) + })?; decimal_map.insert(name.to_string(), (length, scale)); } } + let mut fields_name = schema.names_str(); + if !is_append_only { + fields_name.push(STARROCKS_DELETE_SIGN); + }; - let builder = HeaderBuilder::new() + let header = HeaderBuilder::new() .add_common_header() .set_user_password(config.common.user.clone(), config.common.password.clone()) - .add_json_format(); - let header = if !is_append_only { - let mut fields_name = schema.names_str(); - fields_name.push(STARROCKS_DELETE_SIGN); - builder.set_columns_name(fields_name).build() - } else { - builder.build() - }; + .add_json_format() + .set_partial_update(config.common.partial_update.clone()) + .set_columns_name(fields_name) + .build(); let starrocks_insert_builder = InserterInnerBuilder::new( format!("http://{}:{}", config.common.host, config.common.http_port), @@ -350,7 +367,7 @@ impl StarrocksSinkWriter { inserter_innet_builder: starrocks_insert_builder, is_append_only, client: None, - row_encoder: JsonEncoder::new_with_doris( + row_encoder: JsonEncoder::new_with_starrocks( schema, None, TimestampHandlingMode::String, @@ -386,7 +403,7 @@ impl StarrocksSinkWriter { Value::String("0".to_string()), ); let row_json_string = serde_json::to_string(&row_json_value).map_err(|e| { - SinkError::Starrocks(format!("Json derialize error {:?}", e)) + SinkError::Starrocks(format!("Json derialize error: {}", e.as_report())) })?; self.client .as_mut() @@ -403,7 +420,7 @@ impl StarrocksSinkWriter { Value::String("1".to_string()), ); let row_json_string = serde_json::to_string(&row_json_value).map_err(|e| { - SinkError::Starrocks(format!("Json derialize error {:?}", e)) + SinkError::Starrocks(format!("Json derialize error: {}", e.as_report())) })?; self.client .as_mut() @@ -421,7 +438,7 @@ impl StarrocksSinkWriter { Value::String("0".to_string()), ); let row_json_string = serde_json::to_string(&row_json_value).map_err(|e| { - SinkError::Starrocks(format!("Json derialize error {:?}", e)) + SinkError::Starrocks(format!("Json derialize error: {}", e.as_report())) })?; self.client .as_mut() diff --git a/src/connector/src/sink/writer.rs b/src/connector/src/sink/writer.rs index fdfe1acd4301e..1d8142061f35b 100644 --- a/src/connector/src/sink/writer.rs +++ b/src/connector/src/sink/writer.rs @@ -22,7 +22,7 @@ use futures::future::{select, Either}; use futures::TryFuture; use risingwave_common::array::StreamChunk; use risingwave_common::buffer::Bitmap; -use risingwave_common::util::drop_either_future; +use rw_futures_util::drop_either_future; use crate::sink::encoder::SerTo; use crate::sink::formatter::SinkFormatter; diff --git a/src/connector/src/source/base.rs b/src/connector/src/source/base.rs index 2016fee6de60d..86374f68e7450 100644 --- a/src/connector/src/source/base.rs +++ b/src/connector/src/source/base.rs @@ -15,7 +15,7 @@ use std::collections::{BTreeMap, HashMap}; use std::sync::Arc; -use anyhow::{anyhow, Result}; +use anyhow::anyhow; use async_trait::async_trait; use aws_sdk_s3::types::Object; use bytes::Bytes; @@ -25,8 +25,9 @@ use futures::Stream; use itertools::Itertools; use parking_lot::Mutex; use risingwave_common::array::StreamChunk; +use risingwave_common::bail; use risingwave_common::catalog::TableId; -use risingwave_common::error::{ErrorSuppressor, RwError}; +use risingwave_common::error::ErrorSuppressor; use risingwave_common::metrics::GLOBAL_ERROR_METRICS; use risingwave_common::types::{JsonbVal, Scalar}; use risingwave_pb::catalog::{PbSource, PbStreamSourceInfo}; @@ -34,6 +35,7 @@ use risingwave_pb::plan_common::ExternalTableDesc; use risingwave_pb::source::ConnectorSplit; use risingwave_rpc_client::ConnectorClient; use serde::de::DeserializeOwned; +use thiserror_ext::AsReport; use super::cdc::DebeziumCdcMeta; use super::datagen::DatagenMeta; @@ -44,6 +46,7 @@ use super::kinesis::KinesisMeta; use super::monitor::SourceMetrics; use super::nexmark::source::message::NexmarkMeta; use super::{GCS_CONNECTOR, OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR}; +use crate::error::ConnectorResult as Result; use crate::parser::ParserConfig; pub(crate) use crate::source::common::CommonSplitReader; use crate::source::filesystem::FsPageItem; @@ -68,7 +71,9 @@ pub trait TryFromHashmap: Sized + UnknownFields { /// Each instance should add a `#[derive(with_options::WithOptions)]` marker. pub trait SourceProperties: TryFromHashmap + Clone + WithOptions { const SOURCE_NAME: &'static str; - type Split: SplitMetaData + TryFrom + Into; + type Split: SplitMetaData + + TryFrom + + Into; type SplitEnumerator: SplitEnumerator; type SplitReader: SplitReader; @@ -84,16 +89,16 @@ pub trait UnknownFields { impl TryFromHashmap for P { fn try_from_hashmap(props: HashMap, deny_unknown_fields: bool) -> Result { - let json_value = serde_json::to_value(props).map_err(|e| anyhow!(e))?; - let res = serde_json::from_value::

(json_value).map_err(|e| anyhow!(e.to_string()))?; + let json_value = serde_json::to_value(props)?; + let res = serde_json::from_value::

(json_value)?; if !deny_unknown_fields || res.unknown_fields().is_empty() { Ok(res) } else { - Err(anyhow!( + bail!( "Unknown fields in the WITH clause: {:?}", res.unknown_fields() - )) + ) } } } @@ -132,12 +137,15 @@ pub struct SourceCtrlOpts { // comes from developer::stream_chunk_size in stream scenario and developer::batch_chunk_size // in batch scenario pub chunk_size: usize, + /// Rate limit of source + pub rate_limit: Option, } impl Default for SourceCtrlOpts { fn default() -> Self { Self { chunk_size: MAX_CHUNK_SIZE, + rate_limit: None, } } } @@ -149,7 +157,7 @@ pub struct SourceEnumeratorContext { pub connector_client: Option, } -#[derive(Clone, Copy, Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct SourceEnumeratorInfo { pub source_id: u32, } @@ -172,6 +180,7 @@ impl SourceContext { source_ctrl_opts: SourceCtrlOpts, connector_client: Option, connector_props: ConnectorProperties, + source_name: String, ) -> Self { Self { connector_client, @@ -179,6 +188,7 @@ impl SourceContext { actor_id, source_id: table_id, fragment_id, + source_name, }, metrics, source_ctrl_opts, @@ -196,6 +206,7 @@ impl SourceContext { connector_client: Option, error_suppressor: Arc>, connector_props: ConnectorProperties, + source_name: String, ) -> Self { let mut ctx = Self::new( actor_id, @@ -205,16 +216,17 @@ impl SourceContext { source_ctrl_opts, connector_client, connector_props, + source_name, ); ctx.error_suppressor = Some(error_suppressor); ctx } - pub(crate) fn report_user_source_error(&self, e: RwError) { + pub(crate) fn report_user_source_error(&self, e: &(impl AsReport + ?Sized)) { if self.source_info.fragment_id == u32::MAX { return; } - let mut err_str = e.inner().to_string(); + let mut err_str = e.to_report_string(); if let Some(suppressor) = &self.error_suppressor && suppressor.lock().suppress_error(&err_str) { @@ -235,12 +247,13 @@ impl SourceContext { } } -#[derive(Clone, Copy, Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct SourceInfo { pub actor_id: u32, pub source_id: TableId, // There should be a 1-1 mapping between `source_id` & `fragment_id` pub fragment_id: u32, + pub source_name: String, } #[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] @@ -248,6 +261,7 @@ pub enum SourceFormat { #[default] Invalid, Native, + None, Debezium, DebeziumMongo, Maxwell, @@ -261,6 +275,7 @@ pub enum SourceEncode { #[default] Invalid, Native, + None, Avro, Csv, Protobuf, @@ -304,8 +319,8 @@ pub fn extract_source_struct(info: &PbStreamSourceInfo) -> Result }; return Ok(SourceStruct::new(format, encode)); } - let source_format = info.get_format().map_err(|e| anyhow!("{e:?}"))?; - let source_encode = info.get_row_encode().map_err(|e| anyhow!("{e:?}"))?; + let source_format = info.get_format()?; + let source_encode = info.get_row_encode()?; let (format, encode) = match (source_format, source_encode) { (PbFormatType::Plain, PbEncodeType::Json) => (SourceFormat::Plain, SourceEncode::Json), (PbFormatType::Plain, PbEncodeType::Protobuf) => { @@ -321,6 +336,7 @@ pub fn extract_source_struct(info: &PbStreamSourceInfo) -> Result (PbFormatType::Native, PbEncodeType::Native) => { (SourceFormat::Native, SourceEncode::Native) } + (PbFormatType::None, PbEncodeType::None) => (SourceFormat::None, SourceEncode::None), (PbFormatType::Debezium, PbEncodeType::Avro) => { (SourceFormat::Debezium, SourceEncode::Avro) } @@ -331,41 +347,22 @@ pub fn extract_source_struct(info: &PbStreamSourceInfo) -> Result } (PbFormatType::Plain, PbEncodeType::Bytes) => (SourceFormat::Plain, SourceEncode::Bytes), (format, encode) => { - return Err(anyhow!( + bail!( "Unsupported combination of format {:?} and encode {:?}", format, encode - )); + ); } }; Ok(SourceStruct::new(format, encode)) } -pub type BoxSourceStream = BoxStream<'static, Result>>; - -pub trait SourceWithStateStream = - Stream> + Send + 'static; -pub type BoxSourceWithStateStream = BoxStream<'static, Result>; -pub type BoxTryStream = BoxStream<'static, Result>; - -/// [`StreamChunkWithState`] returns stream chunk together with offset for each split. In the -/// current design, one connector source can have multiple split reader. The keys are unique -/// `split_id` and values are the latest offset for each split. -#[derive(Clone, Debug, PartialEq)] -pub struct StreamChunkWithState { - pub chunk: StreamChunk, - pub split_offset_mapping: Option>, -} +pub type BoxSourceStream = BoxStream<'static, crate::error::ConnectorResult>>; -/// The `split_offset_mapping` field is unused for the table source, so we implement `From` for it. -impl From for StreamChunkWithState { - fn from(chunk: StreamChunk) -> Self { - Self { - chunk, - split_offset_mapping: None, - } - } -} +pub trait ChunkSourceStream = + Stream> + Send + 'static; +pub type BoxChunkSourceStream = BoxStream<'static, crate::error::ConnectorResult>; +pub type BoxTryStream = BoxStream<'static, crate::error::ConnectorResult>; /// [`SplitReader`] is a new abstraction of the external connector read interface which is /// responsible for parsing, it is used to read messages from the outside and transform them into a @@ -381,9 +378,9 @@ pub trait SplitReader: Sized + Send { parser_config: ParserConfig, source_ctx: SourceContextRef, columns: Option>, - ) -> Result; + ) -> crate::error::ConnectorResult; - fn into_stream(self) -> BoxSourceWithStateStream; + fn into_stream(self) -> BoxChunkSourceStream; } for_all_sources!(impl_connector_properties); @@ -437,7 +434,7 @@ impl ConnectorProperties { PropType, PropType::try_from_hashmap(with_properties, deny_unknown_fields) .map(ConnectorProperties::from), - |other| Err(anyhow!("connector '{}' is not supported", other)) + |other| bail!("connector '{}' is not supported", other) ) } @@ -456,6 +453,8 @@ impl ConnectorProperties { pub fn support_multiple_splits(&self) -> bool { matches!(self, ConnectorProperties::Kafka(_)) + || matches!(self, ConnectorProperties::OpendalS3(_)) + || matches!(self, ConnectorProperties::Gcs(_)) } } @@ -473,7 +472,7 @@ impl From<&SplitImpl> for ConnectorSplit { } impl TryFrom<&ConnectorSplit> for SplitImpl { - type Error = anyhow::Error; + type Error = crate::error::ConnectorError; fn try_from(split: &ConnectorSplit) -> std::result::Result { match_source_name_str!( @@ -485,7 +484,7 @@ impl TryFrom<&ConnectorSplit> for SplitImpl { ) .map(Into::into) }, - |other| Err(anyhow!("connector '{}' is not supported", other)) + |other| bail!("connector '{}' is not supported", other) ) } } @@ -514,7 +513,7 @@ impl SplitImpl { split_type.to_lowercase().as_str(), PropType, ::Split::restore_from_json(value).map(Into::into), - |other| Err(anyhow!("connector '{}' is not supported", other)) + |other| bail!("connector '{}' is not supported", other) ) } } @@ -632,7 +631,7 @@ pub trait SplitMetaData: Sized { fn encode_to_json(&self) -> JsonbVal; fn restore_from_json(value: JsonbVal) -> Result; - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()>; + fn update_with_offset(&mut self, start_offset: String) -> crate::error::ConnectorResult<()>; } /// [`ConnectorState`] maintains the consuming splits' info. In specific split readers, @@ -658,7 +657,7 @@ mod tests { use nexmark::event::EventType; use super::*; - use crate::source::cdc::{DebeziumCdcSplit, MySqlCdcSplit}; + use crate::source::cdc::{DebeziumCdcSplit, Mysql}; use crate::source::kafka::KafkaSplit; #[test] @@ -676,8 +675,7 @@ mod tests { #[test] fn test_cdc_split_state() -> Result<()> { let offset_str = "{\"sourcePartition\":{\"server\":\"RW_CDC_mydb.products\"},\"sourceOffset\":{\"transaction_id\":null,\"ts_sec\":1670407377,\"file\":\"binlog.000001\",\"pos\":98587,\"row\":2,\"server_id\":1,\"event\":2}}"; - let mysql_split = MySqlCdcSplit::new(1001, offset_str.to_string()); - let split = DebeziumCdcSplit::new(Some(mysql_split), None); + let split = DebeziumCdcSplit::::new(1001, Some(offset_str.to_string()), None); let split_impl = SplitImpl::MysqlCdc(split); let encoded_split = split_impl.encode_to_bytes(); let restored_split_impl = SplitImpl::restore_from_bytes(encoded_split.as_ref())?; @@ -733,8 +731,11 @@ mod tests { let props = ConnectorProperties::extract(props, true).unwrap(); if let ConnectorProperties::Kafka(k) = props { - assert!(k.common.broker_rewrite_map.is_some()); - println!("{:?}", k.common.broker_rewrite_map); + let hashmap: HashMap = hashmap! { + "b-1:9092".to_string() => "dns-1".to_string(), + "b-2:9092".to_string() => "dns-2".to_string(), + }; + assert_eq!(k.privatelink_common.broker_rewrite_map, Some(hashmap)); } else { panic!("extract kafka config failed"); } diff --git a/src/connector/src/source/cdc/enumerator/mod.rs b/src/connector/src/source/cdc/enumerator/mod.rs index 1664640eef03f..a3e891735d3f2 100644 --- a/src/connector/src/source/cdc/enumerator/mod.rs +++ b/src/connector/src/source/cdc/enumerator/mod.rs @@ -16,7 +16,7 @@ use std::marker::PhantomData; use std::ops::Deref; use std::str::FromStr; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use itertools::Itertools; use prost::Message; @@ -27,9 +27,9 @@ use risingwave_pb::connector_service::{ SourceCommonParam, SourceType, ValidateSourceRequest, ValidateSourceResponse, }; +use crate::error::ConnectorResult; use crate::source::cdc::{ - CdcProperties, CdcSourceTypeTrait, CdcSplitBase, Citus, DebeziumCdcSplit, MySqlCdcSplit, Mysql, - Postgres, PostgresCdcSplit, + CdcProperties, CdcSourceTypeTrait, Citus, DebeziumCdcSplit, Mongodb, Mysql, Postgres, }; use crate::source::{SourceEnumeratorContextRef, SplitEnumerator}; @@ -54,7 +54,7 @@ where async fn new( props: CdcProperties, context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> ConnectorResult { let server_addrs = props .properties .get(DATABASE_SERVERS_KEY) @@ -72,7 +72,7 @@ where ); let source_id = context.info.source_id; - tokio::task::spawn_blocking(move || { + tokio::task::spawn_blocking(move || -> anyhow::Result<()> { let mut env = JVM.get_or_init()?.attach_current_thread()?; let validate_source_request = ValidateSourceRequest { @@ -100,18 +100,14 @@ where .deref(), )?; - validate_source_response.error.map_or_else( - || Ok(()), - |err| { - Err(anyhow!(format!( - "source cannot pass validation: {}", - err.error_message - ))) - }, - ) + if let Some(error) = validate_source_response.error { + return Err(anyhow!(error.error_message).context("source cannot pass validation")); + } + + Ok(()) }) .await - .map_err(|e| anyhow!("failed to validate source: {:?}", e))??; + .context("failed to validate source")??; tracing::debug!("validate cdc source properties success"); Ok(Self { @@ -121,7 +117,7 @@ where }) } - async fn list_splits(&mut self) -> anyhow::Result>> { + async fn list_splits(&mut self) -> ConnectorResult>> { Ok(self.list_cdc_splits()) } } @@ -136,15 +132,11 @@ impl ListCdcSplits for DebeziumSplitEnumerator { fn list_cdc_splits(&mut self) -> Vec> { // CDC source only supports single split - let split = MySqlCdcSplit { - inner: CdcSplitBase::new(self.source_id, None), - }; - let dbz_split = DebeziumCdcSplit { - mysql_split: Some(split), - pg_split: None, - _phantom: PhantomData, - }; - vec![dbz_split] + vec![DebeziumCdcSplit::::new( + self.source_id, + None, + None, + )] } } @@ -152,16 +144,12 @@ impl ListCdcSplits for DebeziumSplitEnumerator { type CdcSourceType = Postgres; fn list_cdc_splits(&mut self) -> Vec> { - let split = PostgresCdcSplit { - inner: CdcSplitBase::new(self.source_id, None), - server_addr: None, - }; - let dbz_split = DebeziumCdcSplit { - mysql_split: None, - pg_split: Some(split), - _phantom: Default::default(), - }; - vec![dbz_split] + // CDC source only supports single split + vec![DebeziumCdcSplit::::new( + self.source_id, + None, + None, + )] } } @@ -173,16 +161,24 @@ impl ListCdcSplits for DebeziumSplitEnumerator { .iter() .enumerate() .map(|(id, addr)| { - let split = PostgresCdcSplit { - inner: CdcSplitBase::new(id as u32, None), - server_addr: Some(addr.to_string()), - }; - DebeziumCdcSplit { - mysql_split: None, - pg_split: Some(split), - _phantom: Default::default(), - } + DebeziumCdcSplit::::new( + id as u32, + None, + Some(addr.to_string()), + ) }) .collect_vec() } } +impl ListCdcSplits for DebeziumSplitEnumerator { + type CdcSourceType = Mongodb; + + fn list_cdc_splits(&mut self) -> Vec> { + // CDC source only supports single split + vec![DebeziumCdcSplit::::new( + self.source_id, + None, + None, + )] + } +} diff --git a/src/connector/src/source/cdc/external/mock_external_table.rs b/src/connector/src/source/cdc/external/mock_external_table.rs index cf3eda6d92104..d7c39e2a9aa8b 100644 --- a/src/connector/src/source/cdc/external/mock_external_table.rs +++ b/src/connector/src/source/cdc/external/mock_external_table.rs @@ -19,9 +19,9 @@ use futures_async_stream::try_stream; use risingwave_common::row::OwnedRow; use risingwave_common::types::ScalarImpl; -use crate::error::ConnectorError; +use crate::error::{ConnectorError, ConnectorResult}; use crate::source::cdc::external::{ - CdcOffset, ConnectorResult, ExternalTableReader, MySqlOffset, SchemaTableName, + CdcOffset, CdcOffsetParseFunc, ExternalTableReader, MySqlOffset, SchemaTableName, }; #[derive(Debug)] @@ -38,6 +38,10 @@ impl MockExternalTableReader { } } + pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc { + Box::new(move |_| Ok(CdcOffset::MySql(MySqlOffset::default()))) + } + #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)] async fn snapshot_read_inner(&self) { let snap_idx = self @@ -72,7 +76,7 @@ impl MockExternalTableReader { ]), ]; - let snapshots = vec![snap0, snap1]; + let snapshots = [snap0, snap1]; if snap_idx >= snapshots.len() { return Ok(()); } @@ -102,13 +106,6 @@ impl ExternalTableReader for MockExternalTableReader { } } - fn parse_cdc_offset(&self, offset: &str) -> ConnectorResult { - // same as mysql offset - Ok(CdcOffset::MySql(MySqlOffset::parse_debezium_offset( - offset, - )?)) - } - fn snapshot_read( &self, _table_name: SchemaTableName, diff --git a/src/connector/src/source/cdc/external/mod.rs b/src/connector/src/source/cdc/external/mod.rs index d520522440840..a9808e3a9e1e2 100644 --- a/src/connector/src/source/cdc/external/mod.rs +++ b/src/connector/src/source/cdc/external/mod.rs @@ -17,7 +17,7 @@ mod postgres; use std::collections::HashMap; -use anyhow::anyhow; +use anyhow::Context; use futures::stream::BoxStream; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; @@ -32,13 +32,11 @@ use risingwave_common::types::DataType; use risingwave_common::util::iter_util::ZipEqFast; use serde_derive::{Deserialize, Serialize}; -use crate::error::ConnectorError; +use crate::error::{ConnectorError, ConnectorResult}; use crate::parser::mysql_row_to_owned_row; use crate::source::cdc::external::mock_external_table::MockExternalTableReader; use crate::source::cdc::external::postgres::{PostgresExternalTableReader, PostgresOffset}; -pub type ConnectorResult = std::result::Result; - #[derive(Debug)] pub enum CdcTableType { Undefined, @@ -77,10 +75,7 @@ impl CdcTableType { Self::Postgres => Ok(ExternalTableReaderImpl::Postgres( PostgresExternalTableReader::new(with_properties, schema).await?, )), - _ => bail!(ConnectorError::Config(anyhow!( - "invalid external table type: {:?}", - *self - ))), + _ => bail!("invalid external table type: {:?}", *self), } } } @@ -191,30 +186,29 @@ pub struct DebeziumSourceOffset { impl MySqlOffset { pub fn parse_debezium_offset(offset: &str) -> ConnectorResult { - let dbz_offset: DebeziumOffset = serde_json::from_str(offset).map_err(|e| { - ConnectorError::Internal(anyhow!("invalid upstream offset: {}, error: {}", offset, e)) - })?; + let dbz_offset: DebeziumOffset = serde_json::from_str(offset) + .with_context(|| format!("invalid upstream offset: {}", offset))?; Ok(Self { filename: dbz_offset .source_offset .file - .ok_or_else(|| anyhow!("binlog file not found in offset"))?, + .context("binlog file not found in offset")?, position: dbz_offset .source_offset .pos - .ok_or_else(|| anyhow!("binlog position not found in offset"))?, + .context("binlog position not found in offset")?, }) } } +pub type CdcOffsetParseFunc = Box ConnectorResult + Send>; + pub trait ExternalTableReader { fn get_normalized_table_name(&self, table_name: &SchemaTableName) -> String; async fn current_cdc_offset(&self) -> ConnectorResult; - fn parse_cdc_offset(&self, dbz_offset: &str) -> ConnectorResult; - fn snapshot_read( &self, table_name: SchemaTableName, @@ -268,7 +262,8 @@ impl ExternalTableReader for MySqlExternalTableReader { let row = rs .iter_mut() .exactly_one() - .map_err(|e| ConnectorError::Internal(anyhow!("read binlog error: {}", e)))?; + .ok() + .context("expect exactly one row when reading binlog offset")?; Ok(CdcOffset::MySql(MySqlOffset { filename: row.take("File").unwrap(), @@ -276,12 +271,6 @@ impl ExternalTableReader for MySqlExternalTableReader { })) } - fn parse_cdc_offset(&self, offset: &str) -> ConnectorResult { - Ok(CdcOffset::MySql(MySqlOffset::parse_debezium_offset( - offset, - )?)) - } - fn snapshot_read( &self, table_name: SchemaTableName, @@ -302,9 +291,7 @@ impl MySqlExternalTableReader { let config = serde_json::from_value::( serde_json::to_value(with_properties).unwrap(), ) - .map_err(|e| { - ConnectorError::Config(anyhow!("fail to extract mysql connector properties: {}", e)) - })?; + .context("failed to extract mysql connector properties")?; let database_url = format!( "mysql://{}:{}@{}:{}/{}", @@ -328,6 +315,14 @@ impl MySqlExternalTableReader { }) } + pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc { + Box::new(move |offset| { + Ok(CdcOffset::MySql(MySqlOffset::parse_debezium_offset( + offset, + )?)) + }) + } + #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)] async fn snapshot_read_inner( &self, @@ -405,22 +400,14 @@ impl MySqlExternalTableReader { DataType::Date => Value::from(value.into_date().0), DataType::Time => Value::from(value.into_time().0), DataType::Timestamp => Value::from(value.into_timestamp().0), - _ => { - return Err(ConnectorError::Internal(anyhow!( - "unsupported primary key data type: {}", - ty - ))) - } + _ => bail!("unsupported primary key data type: {}", ty), }; - Ok((pk.clone(), val)) + ConnectorResult::Ok((pk.clone(), val)) } else { - Err(ConnectorError::Internal(anyhow!( - "primary key {} cannot be null", - pk - ))) + bail!("primary key {} cannot be null", pk); } }) - .try_collect()?; + .try_collect::<_, _, ConnectorError>()?; let rs_stream = sql .with(Params::from(params)) @@ -502,14 +489,6 @@ impl ExternalTableReader for ExternalTableReaderImpl { } } - fn parse_cdc_offset(&self, offset: &str) -> ConnectorResult { - match self { - ExternalTableReaderImpl::MySql(mysql) => mysql.parse_cdc_offset(offset), - ExternalTableReaderImpl::Postgres(postgres) => postgres.parse_cdc_offset(offset), - ExternalTableReaderImpl::Mock(mock) => mock.parse_cdc_offset(offset), - } - } - fn snapshot_read( &self, table_name: SchemaTableName, @@ -521,6 +500,16 @@ impl ExternalTableReader for ExternalTableReaderImpl { } impl ExternalTableReaderImpl { + pub fn get_cdc_offset_parser(&self) -> CdcOffsetParseFunc { + match self { + ExternalTableReaderImpl::MySql(_) => MySqlExternalTableReader::get_cdc_offset_parser(), + ExternalTableReaderImpl::Postgres(_) => { + PostgresExternalTableReader::get_cdc_offset_parser() + } + ExternalTableReaderImpl::Mock(_) => MockExternalTableReader::get_cdc_offset_parser(), + } + } + #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)] async fn snapshot_read_inner( &self, @@ -623,6 +612,10 @@ mod tests { let offset = reader.current_cdc_offset().await.unwrap(); println!("BinlogOffset: {:?}", offset); + let off0_str = r#"{ "sourcePartition": { "server": "test" }, "sourceOffset": { "ts_sec": 1670876905, "file": "binlog.000001", "pos": 105622, "snapshot": true }, "isHeartbeat": false }"#; + let parser = MySqlExternalTableReader::get_cdc_offset_parser(); + println!("parsed offset: {:?}", parser(off0_str).unwrap()); + let table_name = SchemaTableName { schema_name: "mytest".to_string(), table_name: "t1".to_string(), diff --git a/src/connector/src/source/cdc/external/postgres.rs b/src/connector/src/source/cdc/external/postgres.rs index f940e34157532..9f9a055fd7d8f 100644 --- a/src/connector/src/source/cdc/external/postgres.rs +++ b/src/connector/src/source/cdc/external/postgres.rs @@ -15,7 +15,7 @@ use std::cmp::Ordering; use std::collections::HashMap; -use anyhow::anyhow; +use anyhow::Context; use futures::stream::BoxStream; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; @@ -24,13 +24,14 @@ use risingwave_common::catalog::Schema; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::DatumRef; use serde_derive::{Deserialize, Serialize}; +use thiserror_ext::AsReport; use tokio_postgres::types::PgLsn; use tokio_postgres::NoTls; -use crate::error::ConnectorError; +use crate::error::{ConnectorError, ConnectorResult}; use crate::parser::postgres_row_to_owned_row; use crate::source::cdc::external::{ - CdcOffset, ConnectorResult, DebeziumOffset, ExternalTableConfig, ExternalTableReader, + CdcOffset, CdcOffsetParseFunc, DebeziumOffset, ExternalTableConfig, ExternalTableReader, SchemaTableName, }; @@ -51,19 +52,18 @@ impl PartialOrd for PostgresOffset { impl PostgresOffset { pub fn parse_debezium_offset(offset: &str) -> ConnectorResult { - let dbz_offset: DebeziumOffset = serde_json::from_str(offset).map_err(|e| { - ConnectorError::Internal(anyhow!("invalid upstream offset: {}, error: {}", offset, e)) - })?; + let dbz_offset: DebeziumOffset = serde_json::from_str(offset) + .with_context(|| format!("invalid upstream offset: {}", offset))?; Ok(Self { txid: dbz_offset .source_offset .txid - .ok_or_else(|| anyhow!("invalid postgres txid"))?, + .context("invalid postgres txid")?, lsn: dbz_offset .source_offset .lsn - .ok_or_else(|| anyhow!("invalid postgres lsn"))?, + .context("invalid postgres lsn")?, }) } } @@ -105,12 +105,6 @@ impl ExternalTableReader for PostgresExternalTableReader { Ok(CdcOffset::Postgres(pg_offset)) } - fn parse_cdc_offset(&self, offset: &str) -> ConnectorResult { - Ok(CdcOffset::Postgres(PostgresOffset::parse_debezium_offset( - offset, - )?)) - } - fn snapshot_read( &self, table_name: SchemaTableName, @@ -131,12 +125,7 @@ impl PostgresExternalTableReader { let config = serde_json::from_value::( serde_json::to_value(properties).unwrap(), ) - .map_err(|e| { - ConnectorError::Config(anyhow!( - "fail to extract postgres connector properties: {}", - e - )) - })?; + .context("failed to extract postgres connector properties")?; let database_url = format!( "postgresql://{}:{}@{}:{}/{}", @@ -147,7 +136,7 @@ impl PostgresExternalTableReader { tokio::spawn(async move { if let Err(e) = connection.await { - tracing::error!("connection error: {}", e); + tracing::error!(error = %e.as_report(), "postgres connection error"); } }); @@ -165,6 +154,14 @@ impl PostgresExternalTableReader { }) } + pub fn get_cdc_offset_parser() -> CdcOffsetParseFunc { + Box::new(move |offset| { + Ok(CdcOffset::Postgres(PostgresOffset::parse_debezium_offset( + offset, + )?)) + }) + } + #[try_stream(boxed, ok = OwnedRow, error = ConnectorError)] async fn snapshot_read_inner( &self, @@ -202,7 +199,7 @@ impl PostgresExternalTableReader { let stream = client.query_raw(&sql, ¶ms).await?; let row_stream = stream.map(|row| { let row = row?; - Ok::<_, anyhow::Error>(postgres_row_to_owned_row(row, &self.rw_schema)) + Ok::<_, crate::error::ConnectorError>(postgres_row_to_owned_row(row, &self.rw_schema)) }); pin_mut!(row_stream); diff --git a/src/connector/src/source/cdc/mod.rs b/src/connector/src/source/cdc/mod.rs index b3a2bc6554c60..b283913b3479f 100644 --- a/src/connector/src/source/cdc/mod.rs +++ b/src/connector/src/source/cdc/mod.rs @@ -28,6 +28,7 @@ use risingwave_pb::plan_common::ExternalTableDesc; use simd_json::prelude::ArrayTrait; pub use source::*; +use crate::error::ConnectorResult; use crate::source::{SourceProperties, SplitImpl, TryFromHashmap}; use crate::{for_all_classified_sources, impl_cdc_source_type}; @@ -35,10 +36,15 @@ pub const CDC_CONNECTOR_NAME_SUFFIX: &str = "-cdc"; pub const CDC_SNAPSHOT_MODE_KEY: &str = "debezium.snapshot.mode"; pub const CDC_SNAPSHOT_BACKFILL: &str = "rw_cdc_backfill"; pub const CDC_SHARING_MODE_KEY: &str = "rw.sharing.mode.enable"; +// User can set snapshot='false' to disable cdc backfill +pub const CDC_BACKFILL_ENABLE_KEY: &str = "snapshot"; +// We enable transaction for shared cdc source by default +pub const CDC_TRANSACTIONAL_KEY: &str = "transactional"; pub const MYSQL_CDC_CONNECTOR: &str = Mysql::CDC_CONNECTOR_NAME; pub const POSTGRES_CDC_CONNECTOR: &str = Postgres::CDC_CONNECTOR_NAME; pub const CITUS_CDC_CONNECTOR: &str = Citus::CDC_CONNECTOR_NAME; +pub const MONGODB_CDC_CONNECTOR: &str = Mongodb::CDC_CONNECTOR_NAME; pub trait CdcSourceTypeTrait: Send + Sync + Clone + 'static { const CDC_CONNECTOR_NAME: &'static str; @@ -53,6 +59,7 @@ impl<'a> From<&'a str> for CdcSourceType { MYSQL_CDC_CONNECTOR => CdcSourceType::Mysql, POSTGRES_CDC_CONNECTOR => CdcSourceType::Postgres, CITUS_CDC_CONNECTOR => CdcSourceType::Citus, + MONGODB_CDC_CONNECTOR => CdcSourceType::Mongodb, _ => CdcSourceType::Unspecified, } } @@ -64,6 +71,7 @@ impl CdcSourceType { CdcSourceType::Mysql => "MySQL", CdcSourceType::Postgres => "Postgres", CdcSourceType::Citus => "Citus", + CdcSourceType::Mongodb => "MongoDB", CdcSourceType::Unspecified => "Unspecified", } } @@ -87,7 +95,7 @@ impl TryFromHashmap for CdcProperties { fn try_from_hashmap( properties: HashMap, _deny_unknown_fields: bool, - ) -> anyhow::Result { + ) -> ConnectorResult { let is_multi_table_shared = properties .get(CDC_SHARING_MODE_KEY) .is_some_and(|v| v == "true"); @@ -103,7 +111,7 @@ impl TryFromHashmap for CdcProperties { impl SourceProperties for CdcProperties where - DebeziumCdcSplit: TryFrom + Into, + DebeziumCdcSplit: TryFrom + Into, DebeziumSplitEnumerator: ListCdcSplits, { type Split = DebeziumCdcSplit; diff --git a/src/connector/src/source/cdc/source/message.rs b/src/connector/src/source/cdc/source/message.rs index 28fe52c52cd1e..5df937a83fbe8 100644 --- a/src/connector/src/source/cdc/source/message.rs +++ b/src/connector/src/source/cdc/source/message.rs @@ -29,7 +29,11 @@ pub struct DebeziumCdcMeta { impl From for SourceMessage { fn from(message: CdcMessage) -> Self { SourceMessage { - key: None, + key: if message.key.is_empty() { + None // only data message has key + } else { + Some(message.key.as_bytes().to_vec()) + }, payload: if message.payload.is_empty() { None // heartbeat message } else { diff --git a/src/connector/src/source/cdc/source/reader.rs b/src/connector/src/source/cdc/source/reader.rs index cb9c7dae3d114..3e63d506fb9bf 100644 --- a/src/connector/src/source/cdc/source/reader.rs +++ b/src/connector/src/source/cdc/source/reader.rs @@ -14,25 +14,29 @@ use std::str::FromStr; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use futures_async_stream::try_stream; use itertools::Itertools; use prost::Message; +use risingwave_common::bail; +use risingwave_common::metrics::GLOBAL_ERROR_METRICS; use risingwave_common::util::addr::HostAddr; use risingwave_jni_core::jvm_runtime::JVM; use risingwave_jni_core::{call_static_method, JniReceiverType, JniSenderType}; use risingwave_pb::connector_service::{ GetEventStreamRequest, GetEventStreamResponse, SourceCommonParam, }; +use thiserror_ext::AsReport; use tokio::sync::mpsc; +use crate::error::{ConnectorError, ConnectorResult}; use crate::parser::ParserConfig; use crate::source::base::SourceMessage; use crate::source::cdc::{CdcProperties, CdcSourceType, CdcSourceTypeTrait, DebeziumCdcSplit}; use crate::source::{ - into_chunk_stream, BoxSourceWithStateStream, Column, CommonSplitReader, SourceContextRef, - SplitId, SplitMetaData, SplitReader, + into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef, SplitId, + SplitMetaData, SplitReader, }; pub struct CdcSplitReader { @@ -64,19 +68,22 @@ impl SplitReader for CdcSplitReader { parser_config: ParserConfig, source_ctx: SourceContextRef, _columns: Option>, - ) -> Result { + ) -> ConnectorResult { assert_eq!(splits.len(), 1); let split = splits.into_iter().next().unwrap(); let split_id = split.id(); let mut properties = conn_props.properties.clone(); + let mut citus_server_addr = None; // For citus, we need to rewrite the `table.name` to capture sharding tables if matches!(T::source_type(), CdcSourceType::Citus) - && let Some(server_addr) = split.server_addr() + && let Some(ref citus_split) = split.citus_split + && let Some(ref server_addr) = citus_split.server_addr { - let host_addr = HostAddr::from_str(&server_addr) - .map_err(|err| anyhow!("invalid server address for cdc split. {}", err))?; + citus_server_addr = Some(server_addr.clone()); + let host_addr = + HostAddr::from_str(server_addr).context("invalid server address for cdc split")?; properties.insert("hostname".to_string(), host_addr.host); properties.insert("port".to_string(), host_addr.port.to_string()); // rewrite table name with suffix to capture all shards in the split @@ -115,10 +122,8 @@ impl SplitReader for CdcSplitReader { let (mut env, get_event_stream_request_bytes) = match result { Ok(inner) => inner, Err(e) => { - let _ = tx.blocking_send(Err(anyhow!( - "err before calling runJniDbzSourceThread: {:?}", - e - ))); + let _ = tx + .blocking_send(Err(e.context("err before calling runJniDbzSourceThread"))); return; } }; @@ -136,7 +141,7 @@ impl SplitReader for CdcSplitReader { tracing::info!(?source_id, "end of jni call runJniDbzSourceThread"); } Err(e) => { - tracing::error!(?source_id, "jni call error: {:?}", e); + tracing::error!(?source_id, error = %e.as_report(), "jni call error"); } } }); @@ -152,13 +157,13 @@ impl SplitReader for CdcSplitReader { } }; if !inited { - return Err(anyhow!("failed to start cdc connector")); + bail!("failed to start cdc connector"); } } tracing::info!(?source_id, "cdc connector started"); match T::source_type() { - CdcSourceType::Mysql | CdcSourceType::Postgres => Ok(Self { + CdcSourceType::Mysql | CdcSourceType::Postgres | CdcSourceType::Mongodb => Ok(Self { source_id: split.split_id() as u64, start_offset: split.start_offset().clone(), server_addr: None, @@ -172,7 +177,7 @@ impl SplitReader for CdcSplitReader { CdcSourceType::Citus => Ok(Self { source_id: split.split_id() as u64, start_offset: split.start_offset().clone(), - server_addr: split.server_addr(), + server_addr: citus_server_addr, conn_props, split_id, snapshot_done: split.snapshot_done(), @@ -186,7 +191,7 @@ impl SplitReader for CdcSplitReader { } } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { let parser_config = self.parser_config.clone(); let source_context = self.source_ctx.clone(); into_chunk_stream(self, parser_config, source_context) @@ -194,7 +199,7 @@ impl SplitReader for CdcSplitReader { } impl CommonSplitReader for CdcSplitReader { - #[try_stream(ok = Vec, error = anyhow::Error)] + #[try_stream(ok = Vec, error = ConnectorError)] async fn into_data_stream(self) { let source_type = T::source_type(); let mut rx = self.rx; @@ -202,16 +207,27 @@ impl CommonSplitReader for CdcSplitReader { let metrics = self.source_ctx.metrics.clone(); while let Some(result) = rx.recv().await { - let GetEventStreamResponse { events, .. } = result?; - tracing::trace!("receive {} cdc events ", events.len()); - metrics - .connector_source_rows_received - .with_label_values(&[source_type.as_str_name(), &source_id]) - .inc_by(events.len() as u64); - let msgs = events.into_iter().map(SourceMessage::from).collect_vec(); - yield msgs; + match result { + Ok(GetEventStreamResponse { events, .. }) => { + tracing::trace!("receive {} cdc events ", events.len()); + metrics + .connector_source_rows_received + .with_label_values(&[source_type.as_str_name(), &source_id]) + .inc_by(events.len() as u64); + let msgs = events.into_iter().map(SourceMessage::from).collect_vec(); + yield msgs; + } + Err(e) => { + GLOBAL_ERROR_METRICS.cdc_source_error.report([ + source_type.as_str_name().into(), + source_id.clone(), + e.to_report_string(), + ]); + Err(e)?; + } + } } - Err(anyhow!("all senders are dropped"))?; + bail!("all senders are dropped"); } } diff --git a/src/connector/src/source/cdc/split.rs b/src/connector/src/source/cdc/split.rs index 4c46b27be75e8..9817512baa078 100644 --- a/src/connector/src/source/cdc/split.rs +++ b/src/connector/src/source/cdc/split.rs @@ -14,12 +14,13 @@ use std::marker::PhantomData; -use anyhow::anyhow; +use anyhow::Context; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::cdc::external::DebeziumOffset; -use crate::source::cdc::CdcSourceTypeTrait; +use crate::source::cdc::{CdcSourceType, CdcSourceTypeTrait}; use crate::source::{SplitId, SplitMetaData}; /// The base states of a CDC split, which will be persisted to checkpoint. @@ -41,6 +42,13 @@ impl CdcSplitBase { } } +trait CdcSplitTrait: Send + Sync { + fn split_id(&self) -> u32; + fn start_offset(&self) -> &Option; + fn is_snapshot_done(&self) -> bool; + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()>; +} + #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)] pub struct MySqlCdcSplit { pub inner: CdcSplitBase, @@ -53,27 +61,45 @@ pub struct PostgresCdcSplit { pub server_addr: Option, } +#[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)] +pub struct MongoDbCdcSplit { + pub inner: CdcSplitBase, +} + impl MySqlCdcSplit { - pub fn new(split_id: u32, start_offset: String) -> MySqlCdcSplit { + pub fn new(split_id: u32, start_offset: Option) -> Self { let split = CdcSplitBase { split_id, - start_offset: Some(start_offset), + start_offset, snapshot_done: false, }; Self { inner: split } } +} + +impl CdcSplitTrait for MySqlCdcSplit { + fn split_id(&self) -> u32 { + self.inner.split_id + } - pub fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn start_offset(&self) -> &Option { + &self.inner.start_offset + } + + fn is_snapshot_done(&self) -> bool { + self.inner.snapshot_done + } + + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { let mut snapshot_done = self.inner.snapshot_done; if !snapshot_done { - let dbz_offset: DebeziumOffset = serde_json::from_str(&start_offset).map_err(|e| { - anyhow!( - "invalid mysql offset: {}, error: {}, split: {}", - start_offset, - e, - self.inner.split_id - ) - })?; + let dbz_offset: DebeziumOffset = + serde_json::from_str(&start_offset).with_context(|| { + format!( + "invalid mysql offset: {}, split: {}", + start_offset, self.inner.split_id + ) + })?; // heartbeat event should not update the `snapshot_done` flag if !dbz_offset.is_heartbeat { @@ -91,29 +117,42 @@ impl MySqlCdcSplit { } impl PostgresCdcSplit { - pub fn new(split_id: u32, start_offset: String) -> PostgresCdcSplit { + pub fn new(split_id: u32, start_offset: Option, server_addr: Option) -> Self { let split = CdcSplitBase { split_id, - start_offset: Some(start_offset), + start_offset, snapshot_done: false, }; Self { inner: split, - server_addr: None, + server_addr, } } +} - pub fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { +impl CdcSplitTrait for PostgresCdcSplit { + fn split_id(&self) -> u32 { + self.inner.split_id + } + + fn start_offset(&self) -> &Option { + &self.inner.start_offset + } + + fn is_snapshot_done(&self) -> bool { + self.inner.snapshot_done + } + + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { let mut snapshot_done = self.inner.snapshot_done; if !snapshot_done { - let dbz_offset: DebeziumOffset = serde_json::from_str(&start_offset).map_err(|e| { - anyhow!( - "invalid postgres offset: {}, error: {}, split: {}", - start_offset, - e, - self.inner.split_id - ) - })?; + let dbz_offset: DebeziumOffset = + serde_json::from_str(&start_offset).with_context(|| { + format!( + "invalid postgres offset: {}, split: {}", + start_offset, self.inner.split_id + ) + })?; // heartbeat event should not update the `snapshot_done` flag if !dbz_offset.is_heartbeat { @@ -130,92 +169,164 @@ impl PostgresCdcSplit { } } +impl MongoDbCdcSplit { + pub fn new(split_id: u32, start_offset: Option) -> Self { + let split = CdcSplitBase { + split_id, + start_offset, + snapshot_done: false, + }; + Self { inner: split } + } +} + +impl CdcSplitTrait for MongoDbCdcSplit { + fn split_id(&self) -> u32 { + self.inner.split_id + } + + fn start_offset(&self) -> &Option { + &self.inner.start_offset + } + + fn is_snapshot_done(&self) -> bool { + self.inner.snapshot_done + } + + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { + let mut snapshot_done = self.inner.snapshot_done; + // extract snapshot state from debezium offset + if !snapshot_done { + let dbz_offset: DebeziumOffset = + serde_json::from_str(&start_offset).with_context(|| { + format!( + "invalid mongodb offset: {}, split: {}", + start_offset, self.inner.split_id + ) + })?; + + // heartbeat event should not update the `snapshot_done` flag + if !dbz_offset.is_heartbeat { + snapshot_done = match dbz_offset.source_offset.snapshot { + Some(val) => !val, + None => true, + }; + } + } + + self.inner.start_offset = Some(start_offset); + // if snapshot_done is already true, it will remain true + self.inner.snapshot_done = snapshot_done; + Ok(()) + } +} + +/// We use this struct to wrap the specific split, which act as an interface to other modules #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)] pub struct DebeziumCdcSplit { pub mysql_split: Option, - pub pg_split: Option, + + #[serde(rename = "pg_split")] // backward compatibility + pub postgres_split: Option, + pub citus_split: Option, + pub mongodb_split: Option, #[serde(skip)] pub _phantom: PhantomData, } +macro_rules! dispatch_cdc_split_inner { + ($dbz_split:expr, $as_type:tt, {$($cdc_source_type:tt),*}, $body:expr) => { + match T::source_type() { + $( + CdcSourceType::$cdc_source_type => { + $crate::paste! { + $dbz_split.[<$cdc_source_type:lower _split>] + .[]() + .expect(concat!(stringify!([<$cdc_source_type:lower>]), " split must exist")) + .$body + } + } + )* + CdcSourceType::Unspecified => { + unreachable!("invalid debezium split"); + } + } + } +} + +// call corresponding split method of the specific cdc source type +macro_rules! dispatch_cdc_split { + ($dbz_split:expr, $as_type:tt, $body:expr) => { + dispatch_cdc_split_inner!($dbz_split, $as_type, {Mysql, Postgres, Citus, Mongodb}, $body) + } +} + impl SplitMetaData for DebeziumCdcSplit { fn id(&self) -> SplitId { - // TODO: may check T to get the specific cdc type - assert!(self.mysql_split.is_some() || self.pg_split.is_some()); - if let Some(split) = &self.mysql_split { - return format!("{}", split.inner.split_id).into(); - } - if let Some(split) = &self.pg_split { - return format!("{}", split.inner.split_id).into(); - } - unreachable!("invalid split") + format!("{}", self.split_id()).into() } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { - // TODO: may check T to get the specific cdc type - assert!(self.mysql_split.is_some() || self.pg_split.is_some()); - if let Some(split) = &mut self.mysql_split { - split.update_with_offset(start_offset)? - } else if let Some(split) = &mut self.pg_split { - split.update_with_offset(start_offset)? - } - Ok(()) + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { + self.update_with_offset(start_offset) } } impl DebeziumCdcSplit { - pub fn new(mysql_split: Option, pg_split: Option) -> Self { - Self { - mysql_split, - pg_split, + pub fn new(split_id: u32, start_offset: Option, server_addr: Option) -> Self { + let mut ret = Self { + mysql_split: None, + postgres_split: None, + citus_split: None, + mongodb_split: None, _phantom: PhantomData, + }; + match T::source_type() { + CdcSourceType::Mysql => { + let split = MySqlCdcSplit::new(split_id, start_offset); + ret.mysql_split = Some(split); + } + CdcSourceType::Postgres => { + let split = PostgresCdcSplit::new(split_id, start_offset, None); + ret.postgres_split = Some(split); + } + CdcSourceType::Citus => { + let split = PostgresCdcSplit::new(split_id, start_offset, server_addr); + ret.citus_split = Some(split); + } + CdcSourceType::Mongodb => { + let split = MongoDbCdcSplit::new(split_id, start_offset); + ret.mongodb_split = Some(split); + } + CdcSourceType::Unspecified => { + unreachable!("invalid debezium split") + } } + ret } pub fn split_id(&self) -> u32 { - if let Some(split) = &self.mysql_split { - return split.inner.split_id; - } - if let Some(split) = &self.pg_split { - return split.inner.split_id; - } - unreachable!("invalid debezium split") + dispatch_cdc_split!(self, ref, split_id()) } pub fn start_offset(&self) -> &Option { - if let Some(split) = &self.mysql_split { - return &split.inner.start_offset; - } - if let Some(split) = &self.pg_split { - return &split.inner.start_offset; - } - unreachable!("invalid debezium split") + dispatch_cdc_split!(self, ref, start_offset()) } pub fn snapshot_done(&self) -> bool { - if let Some(split) = &self.mysql_split { - return split.inner.snapshot_done; - } - if let Some(split) = &self.pg_split { - return split.inner.snapshot_done; - } - unreachable!("invalid debezium split") + dispatch_cdc_split!(self, ref, is_snapshot_done()) } - pub fn server_addr(&self) -> Option { - if let Some(split) = &self.pg_split { - split.server_addr.clone() - } else { - None - } + pub fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { + dispatch_cdc_split!(self, mut, update_with_offset(start_offset)?); + Ok(()) } } diff --git a/src/connector/src/source/common.rs b/src/connector/src/source/common.rs index 11cbfce5d97f5..c145bd7f403da 100644 --- a/src/connector/src/source/common.rs +++ b/src/connector/src/source/common.rs @@ -14,25 +14,26 @@ use futures::{Stream, StreamExt, TryStreamExt}; use futures_async_stream::try_stream; -use risingwave_common::error::RwError; +use risingwave_common::array::StreamChunk; +use crate::error::{ConnectorError, ConnectorResult}; use crate::parser::ParserConfig; -use crate::source::{SourceContextRef, SourceMessage, SplitReader, StreamChunkWithState}; +use crate::source::{SourceContextRef, SourceMessage, SplitReader}; pub(crate) trait CommonSplitReader: SplitReader + 'static { - fn into_data_stream( - self, - ) -> impl Stream, anyhow::Error>> + Send; + fn into_data_stream(self) -> impl Stream>> + Send; } -#[try_stream(boxed, ok = StreamChunkWithState, error = RwError)] +#[try_stream(boxed, ok = StreamChunk, error = ConnectorError)] pub(crate) async fn into_chunk_stream( reader: impl CommonSplitReader, parser_config: ParserConfig, source_ctx: SourceContextRef, ) { let actor_id = source_ctx.source_info.actor_id.to_string(); + let fragment_id = source_ctx.source_info.fragment_id.to_string(); let source_id = source_ctx.source_info.source_id.to_string(); + let source_name = source_ctx.source_info.source_name.to_string(); let metrics = source_ctx.metrics.clone(); let data_stream = reader.into_data_stream(); @@ -51,7 +52,13 @@ pub(crate) async fn into_chunk_stream( for (split_id, msgs) in by_split_id { metrics .partition_input_count - .with_label_values(&[&actor_id, &source_id, split_id]) + .with_label_values(&[ + &actor_id, + &source_id, + split_id, + &source_name, + &fragment_id, + ]) .inc_by(msgs.len() as u64); let sum_bytes = msgs @@ -61,7 +68,13 @@ pub(crate) async fn into_chunk_stream( metrics .partition_input_bytes - .with_label_values(&[&actor_id, &source_id, split_id]) + .with_label_values(&[ + &actor_id, + &source_id, + split_id, + &source_name, + &fragment_id, + ]) .inc_by(sum_bytes); } }) diff --git a/src/connector/src/source/datagen/enumerator/mod.rs b/src/connector/src/source/datagen/enumerator/mod.rs index 47eb54cf24005..5b5b473656d09 100644 --- a/src/connector/src/source/datagen/enumerator/mod.rs +++ b/src/connector/src/source/datagen/enumerator/mod.rs @@ -12,6 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +use anyhow::Context; use async_trait::async_trait; use crate::source::datagen::{DatagenProperties, DatagenSplit}; @@ -30,13 +31,15 @@ impl SplitEnumerator for DatagenSplitEnumerator { async fn new( properties: DatagenProperties, _context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> crate::error::ConnectorResult { let split_num = properties.split_num.unwrap_or_else(|| "1".to_string()); - let split_num = split_num.parse::()?; + let split_num = split_num + .parse::() + .context("failed to parse datagen split num")?; Ok(Self { split_num }) } - async fn list_splits(&mut self) -> anyhow::Result> { + async fn list_splits(&mut self) -> crate::error::ConnectorResult> { let mut splits = vec![]; for i in 0..self.split_num { splits.push(DatagenSplit { diff --git a/src/connector/src/source/datagen/mod.rs b/src/connector/src/source/datagen/mod.rs index b500281ae17c2..fb766564fb461 100644 --- a/src/connector/src/source/datagen/mod.rs +++ b/src/connector/src/source/datagen/mod.rs @@ -54,7 +54,7 @@ pub struct DatagenProperties { /// datagen will create v1 by self-incrementing from 1 to 1000 /// datagen will create v2 by randomly generating from default_min to default_max #[serde(flatten)] - fields: HashMap, + pub fields: HashMap, } impl SourceProperties for DatagenProperties { diff --git a/src/connector/src/source/datagen/source/generator.rs b/src/connector/src/source/datagen/source/generator.rs index 6bf3b782d3949..1c05c6b4ffc8f 100644 --- a/src/connector/src/source/datagen/source/generator.rs +++ b/src/connector/src/source/datagen/source/generator.rs @@ -13,18 +13,17 @@ // limitations under the License. use std::time::{Duration, SystemTime, UNIX_EPOCH}; -use anyhow::Result; use futures_async_stream::try_stream; -use maplit::hashmap; +use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::error::RwError; use risingwave_common::field_generator::FieldGeneratorImpl; use risingwave_common::row::OwnedRow; -use risingwave_common::types::DataType; +use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_common::util::iter_util::ZipEqFast; +use crate::error::ConnectorResult; use crate::parser::{EncodingProperties, ProtocolProperties, SpecificParserConfig}; -use crate::source::{SourceMessage, SourceMeta, SplitId, StreamChunkWithState}; +use crate::source::{SourceMessage, SourceMeta, SplitId}; pub enum FieldDesc { // field is invisible, generate None @@ -61,7 +60,7 @@ impl DatagenEventGenerator { split_id: SplitId, split_num: u64, split_index: u64, - ) -> Result { + ) -> ConnectorResult { let partition_rows_per_second = if rows_per_second % split_num > split_index { rows_per_second / split_num + 1 } else { @@ -78,7 +77,7 @@ impl DatagenEventGenerator { }) } - #[try_stream(boxed, ok = Vec, error = anyhow::Error)] + #[try_stream(boxed, ok = Vec, error = crate::error::ConnectorError)] pub async fn into_msg_stream(mut self) { let mut interval = tokio::time::interval(Duration::from_secs(1)); const MAX_ROWS_PER_YIELD: u64 = 1024; @@ -158,7 +157,7 @@ impl DatagenEventGenerator { } } - #[try_stream(ok = StreamChunkWithState, error = RwError)] + #[try_stream(ok = StreamChunk, error = crate::error::ConnectorError)] pub async fn into_native_stream(mut self) { let mut interval = tokio::time::interval(Duration::from_secs(1)); const MAX_ROWS_PER_YIELD: u64 = 1024; @@ -167,17 +166,29 @@ impl DatagenEventGenerator { // generate `partition_rows_per_second` rows per second interval.tick().await; let mut rows_generated_this_second = 0; + let mut chunk_builder = + StreamChunkBuilder::new(MAX_ROWS_PER_YIELD as usize, self.data_types.clone()); while rows_generated_this_second < self.partition_rows_per_second { - let mut rows = vec![]; let num_rows_to_generate = std::cmp::min( MAX_ROWS_PER_YIELD, self.partition_rows_per_second - rows_generated_this_second, ); 'outer: for _ in 0..num_rows_to_generate { - let mut row = Vec::with_capacity(self.fields_vec.len()); - for field_generator in &mut self.fields_vec { + let mut row = Vec::with_capacity(self.data_types.len()); + for (field_generator, field_name) in + self.fields_vec.iter_mut().zip_eq_fast(&self.field_names) + { let datum = match field_generator { - FieldDesc::Invisible => None, + // TODO: avoid distinguishing hidden partition/offset columns by name + FieldDesc::Invisible => match field_name.as_str() { + "_rw_datagen_partition" => { + Some(ScalarImpl::Utf8(self.split_id.as_ref().into())) + } + "_rw_datagen_offset" => { + Some(ScalarImpl::Utf8(self.offset.to_string().into_boxed_str())) + } + _ => None, + }, FieldDesc::Visible(field_generator) => { let datum = field_generator.generate_datum(self.offset); if datum.is_none() { @@ -197,20 +208,15 @@ impl DatagenEventGenerator { row.push(datum); } - rows.push((Op::Insert, OwnedRow::new(row))); self.offset += 1; rows_generated_this_second += 1; + if let Some(chunk) = chunk_builder.append_row(Op::Insert, OwnedRow::new(row)) { + yield chunk; + } } - if !rows.is_empty() { - let chunk = StreamChunk::from_rows(&rows, &self.data_types); - let mapping = hashmap! { - self.split_id.clone() => (self.offset - 1).to_string() - }; - yield StreamChunkWithState { - chunk, - split_offset_mapping: Some(mapping), - }; + if let Some(chunk) = chunk_builder.take() { + yield chunk; } if reach_end { diff --git a/src/connector/src/source/datagen/source/reader.rs b/src/connector/src/source/datagen/source/reader.rs index d276e6b414210..0b522c4e7c938 100644 --- a/src/connector/src/source/datagen/source/reader.rs +++ b/src/connector/src/source/datagen/source/reader.rs @@ -14,19 +14,21 @@ use std::collections::HashMap; -use anyhow::{anyhow, Result}; +use anyhow::Context; use async_trait::async_trait; use futures::{Stream, StreamExt, TryStreamExt}; use risingwave_common::field_generator::{FieldGeneratorImpl, VarcharProperty}; +use thiserror_ext::AsReport; use super::generator::DatagenEventGenerator; +use crate::error::{ConnectorResult, ConnectorResult as Result}; use crate::parser::{EncodingProperties, ParserConfig, ProtocolProperties}; use crate::source::data_gen_util::spawn_data_generation_stream; use crate::source::datagen::source::SEQUENCE_FIELD_KIND; use crate::source::datagen::{DatagenProperties, DatagenSplit, FieldDesc}; use crate::source::{ - into_chunk_stream, BoxSourceWithStateStream, Column, CommonSplitReader, DataType, - SourceContextRef, SourceMessage, SplitId, SplitMetaData, SplitReader, + into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, DataType, SourceContextRef, + SourceMessage, SplitId, SplitMetaData, SplitReader, }; pub struct DatagenSplitReader { @@ -138,7 +140,7 @@ impl SplitReader for DatagenSplitReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { // Will buffer at most 4 event chunks. const BUFFER_SIZE: usize = 4; // spawn_data_generation_stream(self.generator.into_native_stream(), BUFFER_SIZE).boxed() @@ -148,17 +150,25 @@ impl SplitReader for DatagenSplitReader { ) { (ProtocolProperties::Native, EncodingProperties::Native) => { let actor_id = self.source_ctx.source_info.actor_id.to_string(); + let fragment_id = self.source_ctx.source_info.fragment_id.to_string(); let source_id = self.source_ctx.source_info.source_id.to_string(); + let source_name = self.source_ctx.source_info.source_name.to_string(); let split_id = self.split_id.to_string(); let metrics = self.source_ctx.metrics.clone(); spawn_data_generation_stream( self.generator .into_native_stream() - .inspect_ok(move |chunk_with_states| { + .inspect_ok(move |stream_chunk| { metrics .partition_input_count - .with_label_values(&[&actor_id, &source_id, &split_id]) - .inc_by(chunk_with_states.chunk.cardinality() as u64); + .with_label_values(&[ + &actor_id, + &source_id, + &split_id, + &source_name, + &fragment_id, + ]) + .inc_by(stream_chunk.cardinality() as u64); }), BUFFER_SIZE, ) @@ -174,7 +184,7 @@ impl SplitReader for DatagenSplitReader { } impl CommonSplitReader for DatagenSplitReader { - fn into_data_stream(self) -> impl Stream, anyhow::Error>> { + fn into_data_stream(self) -> impl Stream>> { // Will buffer at most 4 event chunks. const BUFFER_SIZE: usize = 4; spawn_data_generation_stream(self.generator.into_msg_stream(), BUFFER_SIZE) @@ -201,9 +211,9 @@ fn generator_from_data_type( Ok(seed) => seed ^ split_index, Err(e) => { tracing::warn!( - "cannot parse {:?} to u64 due to {:?}, will use {:?} as random seed", + error = %e.as_report(), + "cannot parse {:?} to u64, will use {:?} as random seed", seed, - e, split_index ); split_index @@ -222,11 +232,10 @@ fn generator_from_data_type( .map(|s| s.to_lowercase()); let basetime = match fields_option_map.get(format!("fields.{}.basetime", name).as_str()) { - Some(base) => { - Some(chrono::DateTime::parse_from_rfc3339(base).map_err(|e| { - anyhow!("cannot parse {:?} to rfc3339 due to {:?}", base, e) - })?) - } + Some(base) => Some( + chrono::DateTime::parse_from_rfc3339(base) + .with_context(|| format!("cannot parse `{base}` to rfc3339"))?, + ), None => None, }; @@ -245,13 +254,15 @@ fn generator_from_data_type( random_seed, ) } + .map_err(Into::into) } DataType::Varchar => { let length_key = format!("fields.{}.length", name); let length_value = fields_option_map .get(&length_key) .map(|s| s.parse::()) - .transpose()?; + .transpose() + .context("failed to parse the length of varchar field")?; Ok(FieldGeneratorImpl::with_varchar( &VarcharProperty::RandomFixedLength(length_value), random_seed, @@ -272,7 +283,7 @@ fn generator_from_data_type( Ok((field_name.to_string(), gen)) }) .collect::>()?; - FieldGeneratorImpl::with_struct_fields(struct_fields) + FieldGeneratorImpl::with_struct_fields(struct_fields).map_err(Into::into) } DataType::List(datatype) => { let length_key = format!("fields.{}.length", name); @@ -285,7 +296,7 @@ fn generator_from_data_type( split_num, offset, )?; - FieldGeneratorImpl::with_list(generator, length_value) + FieldGeneratorImpl::with_list(generator, length_value).map_err(Into::into) } _ => { let kind_key = format!("fields.{}.kind", name); @@ -304,12 +315,14 @@ fn generator_from_data_type( split_num, offset, ) + .map_err(Into::into) } else { let min_key = format!("fields.{}.min", name); let max_key = format!("fields.{}.max", name); let min_value = fields_option_map.get(&min_key).map(|s| s.to_string()); let max_value = fields_option_map.get(&max_key).map(|s| s.to_string()); FieldGeneratorImpl::with_number_random(data_type, min_value, max_value, random_seed) + .map_err(Into::into) } } } @@ -397,7 +410,7 @@ mod tests { .into_stream(); let stream_chunk = reader.next().await.unwrap().unwrap(); - let (op, row) = stream_chunk.chunk.rows().next().unwrap(); + let (op, row) = stream_chunk.rows().next().unwrap(); assert_eq!(op, Op::Insert); assert_eq!(row.datum_at(0), Some(ScalarImpl::Int32(533)).to_datum_ref(),); assert_eq!( diff --git a/src/connector/src/source/datagen/split.rs b/src/connector/src/source/datagen/split.rs index c2e0bea1f8fb4..6d51cfa7d47ae 100644 --- a/src/connector/src/source/datagen/split.rs +++ b/src/connector/src/source/datagen/split.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::base::SplitMetaData; use crate::source::SplitId; @@ -32,15 +32,15 @@ impl SplitMetaData for DatagenSplit { format!("{}-{}", self.split_num, self.split_index).into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { self.start_offset = Some(start_offset.as_str().parse::().unwrap()); Ok(()) } diff --git a/src/connector/src/source/filesystem/file_common.rs b/src/connector/src/source/filesystem/file_common.rs index 3a537509edb88..ccff7315491ba 100644 --- a/src/connector/src/source/filesystem/file_common.rs +++ b/src/connector/src/source/filesystem/file_common.rs @@ -15,12 +15,12 @@ use std::fmt::Debug; use std::hash::Hash; use std::marker::PhantomData; -use anyhow::anyhow; use aws_sdk_s3::types::Object; use risingwave_common::types::{JsonbVal, Timestamptz}; use serde::{Deserialize, Serialize}; use super::opendal_source::OpendalSource; +use crate::error::ConnectorResult; use crate::source::{SplitId, SplitMetaData}; /// [`FsSplit`] Describes a file or a split of a file. A file is a generic concept, @@ -47,15 +47,15 @@ impl SplitMetaData for FsSplit { self.name.as_str().into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { let offset = start_offset.parse().unwrap(); self.offset = offset; Ok(()) @@ -98,15 +98,15 @@ impl SplitMetaData for OpendalFsSplit { self.name.as_str().into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { let offset = start_offset.parse().unwrap(); self.offset = offset; Ok(()) diff --git a/src/connector/src/source/filesystem/nd_streaming.rs b/src/connector/src/source/filesystem/nd_streaming.rs index 7eb8a84c503ad..711e1a15e981c 100644 --- a/src/connector/src/source/filesystem/nd_streaming.rs +++ b/src/connector/src/source/filesystem/nd_streaming.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; +use anyhow::Context as _; use bytes::BytesMut; use futures::io::Cursor; use futures::AsyncBufReadExt; @@ -26,7 +26,7 @@ pub fn need_nd_streaming(encode_config: &EncodingProperties) -> bool { || matches!(encode_config, EncodingProperties::Csv(_)) } -#[try_stream(boxed, ok = Vec, error = anyhow::Error)] +#[try_stream(boxed, ok = Vec, error = crate::error::ConnectorError)] /// This function splits a byte stream by the newline separator "(\r)\n" into a message stream. /// It can be difficult to split and compute offsets correctly when the bytes are received in /// chunks. There are two cases to consider: @@ -50,7 +50,7 @@ pub async fn split_stream(data_stream: BoxSourceStream) { .map(|msg| (msg.offset.clone(), msg.split_id.clone(), msg.meta.clone())) .unwrap(); - let mut offset: usize = offset.parse()?; + let mut offset: usize = offset.parse().context("failed to parse the offset")?; let mut buf = BytesMut::new(); for msg in batch { let payload = msg.payload.unwrap_or_default(); @@ -108,7 +108,7 @@ pub async fn split_stream(data_stream: BoxSourceStream) { last_message = msgs.pop(); } } - Err(e) => return Err(anyhow!(e)), + Err(e) => return Err(e.into()), } line_cnt += 1; diff --git a/src/connector/src/source/filesystem/opendal_source/gcs_source.rs b/src/connector/src/source/filesystem/opendal_source/gcs_source.rs index d6f7b44bff591..01594af4e4bad 100644 --- a/src/connector/src/source/filesystem/opendal_source/gcs_source.rs +++ b/src/connector/src/source/filesystem/opendal_source/gcs_source.rs @@ -21,11 +21,12 @@ use opendal::Operator; use super::opendal_enumerator::OpendalEnumerator; use super::{GcsProperties, OpendalSource}; +use crate::error::ConnectorResult; use crate::source::filesystem::s3::enumerator::get_prefix; impl OpendalEnumerator { /// create opendal gcs source. - pub fn new_gcs_source(gcs_properties: GcsProperties) -> anyhow::Result { + pub fn new_gcs_source(gcs_properties: GcsProperties) -> ConnectorResult { // Create gcs builder. let mut builder = Gcs::default(); diff --git a/src/connector/src/source/filesystem/opendal_source/mod.rs b/src/connector/src/source/filesystem/opendal_source/mod.rs index e0c5a22f1fd90..15371a0da90a6 100644 --- a/src/connector/src/source/filesystem/opendal_source/mod.rs +++ b/src/connector/src/source/filesystem/opendal_source/mod.rs @@ -27,6 +27,7 @@ use self::opendal_enumerator::OpendalEnumerator; use self::opendal_reader::OpendalReader; use super::s3::S3PropertiesCommon; use super::OpendalFsSplit; +use crate::error::ConnectorResult; use crate::source::{SourceProperties, UnknownFields}; pub const GCS_CONNECTOR: &str = "gcs"; @@ -71,7 +72,7 @@ impl SourceProperties for GcsProperties { pub trait OpendalSource: Send + Sync + 'static + Clone + PartialEq { type Properties: SourceProperties + Send + Sync; - fn new_enumerator(properties: Self::Properties) -> anyhow::Result>; + fn new_enumerator(properties: Self::Properties) -> ConnectorResult>; } #[derive(Debug, Clone, Copy, PartialEq, Eq)] @@ -80,7 +81,7 @@ pub struct OpendalS3; impl OpendalSource for OpendalS3 { type Properties = OpendalS3Properties; - fn new_enumerator(properties: Self::Properties) -> anyhow::Result> { + fn new_enumerator(properties: Self::Properties) -> ConnectorResult> { OpendalEnumerator::new_s3_source(properties.s3_properties, properties.assume_role) } } @@ -91,7 +92,7 @@ pub struct OpendalGcs; impl OpendalSource for OpendalGcs { type Properties = GcsProperties; - fn new_enumerator(properties: Self::Properties) -> anyhow::Result> { + fn new_enumerator(properties: Self::Properties) -> ConnectorResult> { OpendalEnumerator::new_gcs_source(properties) } } @@ -102,7 +103,7 @@ pub struct OpendalPosixFs; impl OpendalSource for OpendalPosixFs { type Properties = PosixFsProperties; - fn new_enumerator(properties: Self::Properties) -> anyhow::Result> { + fn new_enumerator(properties: Self::Properties) -> ConnectorResult> { OpendalEnumerator::new_posix_fs_source(properties) } } diff --git a/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs b/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs index 318467eea6069..96646ade0e1df 100644 --- a/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs +++ b/src/connector/src/source/filesystem/opendal_source/opendal_enumerator.rs @@ -22,6 +22,7 @@ use opendal::{Metakey, Operator}; use risingwave_common::types::Timestamptz; use super::OpendalSource; +use crate::error::ConnectorResult; use crate::source::filesystem::{FsPageItem, OpendalFsSplit}; use crate::source::{SourceEnumeratorContextRef, SplitEnumerator}; @@ -42,11 +43,11 @@ impl SplitEnumerator for OpendalEnumerator { async fn new( properties: Src::Properties, _context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> ConnectorResult { Src::new_enumerator(properties) } - async fn list_splits(&mut self) -> anyhow::Result>> { + async fn list_splits(&mut self) -> ConnectorResult>> { let empty_split: OpendalFsSplit = OpendalFsSplit::empty_split(); Ok(vec![empty_split]) @@ -54,7 +55,7 @@ impl SplitEnumerator for OpendalEnumerator { } impl OpendalEnumerator { - pub async fn list(&self) -> anyhow::Result { + pub async fn list(&self) -> ConnectorResult { let prefix = match &self.prefix { Some(prefix) => prefix, None => "", @@ -100,4 +101,4 @@ impl OpendalEnumerator { &self.matcher } } -pub type ObjectMetadataIter = BoxStream<'static, anyhow::Result>; +pub type ObjectMetadataIter = BoxStream<'static, ConnectorResult>; diff --git a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs index 57dfa7396128f..99173327be76f 100644 --- a/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs +++ b/src/connector/src/source/filesystem/opendal_source/opendal_reader.rs @@ -12,23 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{Ok, Result}; use async_trait::async_trait; use futures::TryStreamExt; use futures_async_stream::try_stream; use opendal::Operator; -use risingwave_common::error::RwError; +use risingwave_common::array::StreamChunk; use tokio::io::BufReader; use tokio_util::io::{ReaderStream, StreamReader}; use super::opendal_enumerator::OpendalEnumerator; use super::OpendalSource; +use crate::error::ConnectorResult; use crate::parser::{ByteStreamSourceParserImpl, ParserConfig}; use crate::source::filesystem::nd_streaming::need_nd_streaming; use crate::source::filesystem::{nd_streaming, OpendalFsSplit}; use crate::source::{ - BoxSourceWithStateStream, Column, SourceContextRef, SourceMessage, SourceMeta, SplitMetaData, - SplitReader, StreamChunkWithState, + BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta, SplitMetaData, + SplitReader, }; const MAX_CHANNEL_BUFFER_SIZE: usize = 2048; @@ -51,7 +51,7 @@ impl SplitReader for OpendalReader { parser_config: ParserConfig, source_ctx: SourceContextRef, _columns: Option>, - ) -> Result { + ) -> ConnectorResult { let connector = Src::new_enumerator(properties)?; let opendal_reader = OpendalReader { connector, @@ -62,17 +62,19 @@ impl SplitReader for OpendalReader { Ok(opendal_reader) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { self.into_chunk_stream() } } impl OpendalReader { - #[try_stream(boxed, ok = StreamChunkWithState, error = RwError)] + #[try_stream(boxed, ok = StreamChunk, error = crate::error::ConnectorError)] async fn into_chunk_stream(self) { for split in self.splits { let actor_id = self.source_ctx.source_info.actor_id.to_string(); + let fragment_id = self.source_ctx.source_info.fragment_id.to_string(); let source_id = self.source_ctx.source_info.source_id.to_string(); + let source_name = self.source_ctx.source_info.source_name.to_string(); let source_ctx = self.source_ctx.clone(); let split_id = split.id(); @@ -93,21 +95,29 @@ impl OpendalReader { self.source_ctx .metrics .partition_input_count - .with_label_values(&[&actor_id, &source_id, &split_id]) - .inc_by(msg.chunk.cardinality() as u64); + .with_label_values(&[ + &actor_id, + &source_id, + &split_id, + &source_name, + &fragment_id, + ]) + .inc_by(msg.cardinality() as u64); yield msg; } } } - #[try_stream(boxed, ok = Vec, error = anyhow::Error)] + #[try_stream(boxed, ok = Vec, error = crate::error::ConnectorError)] pub async fn stream_read_object( op: Operator, split: OpendalFsSplit, source_ctx: SourceContextRef, ) { let actor_id = source_ctx.source_info.actor_id.to_string(); + let fragment_id = source_ctx.source_info.fragment_id.to_string(); let source_id = source_ctx.source_info.source_id.to_string(); + let source_name = source_ctx.source_info.source_name.to_string(); let max_chunk_size = source_ctx.source_ctrl_opts.chunk_size; let split_id = split.id(); @@ -141,11 +151,18 @@ impl OpendalReader { offset += len; batch_size += len; batch.push(msg); + if batch.len() >= max_chunk_size { source_ctx .metrics .partition_input_bytes - .with_label_values(&[&actor_id, &source_id, &split_id]) + .with_label_values(&[ + &actor_id, + &source_id, + &split_id, + &source_name, + &fragment_id, + ]) .inc_by(batch_size as u64); let yield_batch = std::mem::take(&mut batch); batch_size = 0; @@ -156,7 +173,7 @@ impl OpendalReader { source_ctx .metrics .partition_input_bytes - .with_label_values(&[&actor_id, &source_id, &split_id]) + .with_label_values(&[&actor_id, &source_id, &split_id, &source_name, &fragment_id]) .inc_by(batch_size as u64); yield batch; } diff --git a/src/connector/src/source/filesystem/opendal_source/posix_fs_source.rs b/src/connector/src/source/filesystem/opendal_source/posix_fs_source.rs index 748230ba5a16e..3a4fb7fcfeaa7 100644 --- a/src/connector/src/source/filesystem/opendal_source/posix_fs_source.rs +++ b/src/connector/src/source/filesystem/opendal_source/posix_fs_source.rs @@ -21,6 +21,7 @@ use opendal::Operator; use super::opendal_enumerator::OpendalEnumerator; use super::{OpendalSource, PosixFsProperties}; +use crate::error::ConnectorResult; // Posix fs source should only be used for testing. // For a single-CN cluster, the behavior is well-defined. It will read from the local file system. @@ -28,7 +29,7 @@ use super::{OpendalSource, PosixFsProperties}; impl OpendalEnumerator { /// create opendal posix fs source. - pub fn new_posix_fs_source(posix_fs_properties: PosixFsProperties) -> anyhow::Result { + pub fn new_posix_fs_source(posix_fs_properties: PosixFsProperties) -> ConnectorResult { // Create Fs builder. let mut builder = Fs::default(); diff --git a/src/connector/src/source/filesystem/opendal_source/s3_source.rs b/src/connector/src/source/filesystem/opendal_source/s3_source.rs index ef18ffa4b8fec..f4a548306885a 100644 --- a/src/connector/src/source/filesystem/opendal_source/s3_source.rs +++ b/src/connector/src/source/filesystem/opendal_source/s3_source.rs @@ -21,6 +21,7 @@ use opendal::Operator; use super::opendal_enumerator::OpendalEnumerator; use super::OpendalSource; +use crate::error::ConnectorResult; use crate::source::filesystem::s3::enumerator::get_prefix; use crate::source::filesystem::s3::S3PropertiesCommon; @@ -29,7 +30,7 @@ impl OpendalEnumerator { pub fn new_s3_source( s3_properties: S3PropertiesCommon, assume_role: Option, - ) -> anyhow::Result { + ) -> ConnectorResult { // Create s3 builder. let mut builder = S3::default(); builder.bucket(&s3_properties.bucket_name); diff --git a/src/connector/src/source/filesystem/s3/enumerator.rs b/src/connector/src/source/filesystem/s3/enumerator.rs index 7033ebdaf91ba..63a57b8574b6d 100644 --- a/src/connector/src/source/filesystem/s3/enumerator.rs +++ b/src/connector/src/source/filesystem/s3/enumerator.rs @@ -75,7 +75,7 @@ impl SplitEnumerator for S3SplitEnumerator { async fn new( properties: Self::Properties, _context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> crate::error::ConnectorResult { let config = AwsAuthProps::from(&properties); let sdk_config = config.build_config().await?; let s3_client = s3_client(&sdk_config, Some(default_conn_config())); @@ -98,7 +98,7 @@ impl SplitEnumerator for S3SplitEnumerator { }) } - async fn list_splits(&mut self) -> anyhow::Result> { + async fn list_splits(&mut self) -> crate::error::ConnectorResult> { let mut objects = Vec::new(); loop { let (files, has_finished) = self.get_next_page::().await?; diff --git a/src/connector/src/source/filesystem/s3/source/reader.rs b/src/connector/src/source/filesystem/s3/source/reader.rs index 7d3e638811b63..3f485dc22383a 100644 --- a/src/connector/src/source/filesystem/s3/source/reader.rs +++ b/src/connector/src/source/filesystem/s3/source/reader.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::pin::pin; -use anyhow::{anyhow, Result}; +use anyhow::anyhow; use async_trait::async_trait; use aws_sdk_s3::client as s3_client; use aws_sdk_s3::operation::get_object::GetObjectError; @@ -25,23 +25,21 @@ use aws_smithy_types::body::SdkBody; use aws_smithy_types::byte_stream::ByteStream; use futures_async_stream::try_stream; use io::StreamReader; -use risingwave_common::error::RwError; +use risingwave_common::array::StreamChunk; use tokio::io::BufReader; use tokio_util::io; use tokio_util::io::ReaderStream; use crate::aws_utils::{default_conn_config, s3_client}; use crate::common::AwsAuthProps; +use crate::error::ConnectorResult; use crate::parser::{ByteStreamSourceParserImpl, ParserConfig}; use crate::source::base::{SplitMetaData, SplitReader}; use crate::source::filesystem::file_common::FsSplit; use crate::source::filesystem::nd_streaming; use crate::source::filesystem::nd_streaming::need_nd_streaming; use crate::source::filesystem::s3::S3Properties; -use crate::source::{ - BoxSourceWithStateStream, Column, SourceContextRef, SourceMessage, SourceMeta, - StreamChunkWithState, -}; +use crate::source::{BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SourceMeta}; const MAX_CHANNEL_BUFFER_SIZE: usize = 2048; const STREAM_READER_CAPACITY: usize = 4096; @@ -57,7 +55,7 @@ pub struct S3FileReader { } impl S3FileReader { - #[try_stream(boxed, ok = Vec, error = anyhow::Error)] + #[try_stream(boxed, ok = Vec, error = crate::error::ConnectorError)] pub async fn stream_read_object( client_for_s3: s3_client::Client, bucket_name: String, @@ -65,7 +63,9 @@ impl S3FileReader { source_ctx: SourceContextRef, ) { let actor_id = source_ctx.source_info.actor_id.to_string(); + let fragment_id = source_ctx.source_info.fragment_id.to_string(); let source_id = source_ctx.source_info.source_id.to_string(); + let source_name = source_ctx.source_info.source_name.to_string(); let max_chunk_size = source_ctx.source_ctrl_opts.chunk_size; let split_id = split.id(); @@ -86,11 +86,9 @@ impl S3FileReader { return Ok(()); } Err(e) => { - return Err(anyhow!( - "S3 GetObject from {} error: {}", - bucket_name, - e.to_string() - )); + return Err(anyhow!(e) + .context(format!("S3 GetObject from {bucket_name} error")) + .into()); } }; @@ -124,7 +122,13 @@ impl S3FileReader { source_ctx .metrics .partition_input_bytes - .with_label_values(&[&actor_id, &source_id, &split_id]) + .with_label_values(&[ + &actor_id, + &source_id, + &split_id, + &source_name, + &fragment_id, + ]) .inc_by(batch_size as u64); let yield_batch = std::mem::take(&mut batch); batch_size = 0; @@ -135,7 +139,7 @@ impl S3FileReader { source_ctx .metrics .partition_input_bytes - .with_label_values(&[&actor_id, &source_id, &split_id]) + .with_label_values(&[&actor_id, &source_id, &split_id, &source_name, &fragment_id]) .inc_by(batch_size as u64); yield batch; } @@ -179,7 +183,7 @@ impl SplitReader for S3FileReader { parser_config: ParserConfig, source_ctx: SourceContextRef, _columns: Option>, - ) -> Result { + ) -> ConnectorResult { let config = AwsAuthProps::from(&props); let sdk_config = config.build_config().await?; @@ -199,17 +203,19 @@ impl SplitReader for S3FileReader { Ok(s3_file_reader) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { self.into_chunk_stream() } } impl S3FileReader { - #[try_stream(boxed, ok = StreamChunkWithState, error = RwError)] + #[try_stream(boxed, ok = StreamChunk, error = crate::error::ConnectorError)] async fn into_chunk_stream(self) { for split in self.splits { let actor_id = self.source_ctx.source_info.actor_id.to_string(); + let fragment_id = self.source_ctx.source_info.fragment_id.to_string(); let source_id = self.source_ctx.source_info.source_id.to_string(); + let source_name = self.source_ctx.source_info.source_name.to_string(); let source_ctx = self.source_ctx.clone(); let split_id = split.id(); @@ -234,8 +240,14 @@ impl S3FileReader { self.source_ctx .metrics .partition_input_count - .with_label_values(&[&actor_id, &source_id, &split_id]) - .inc_by(msg.chunk.cardinality() as u64); + .with_label_values(&[ + &actor_id, + &source_id, + &split_id, + &source_name, + &fragment_id, + ]) + .inc_by(msg.cardinality() as u64); yield msg; } } diff --git a/src/connector/src/source/filesystem/s3_v2/lister.rs b/src/connector/src/source/filesystem/s3_v2/lister.rs index d6a6b6c1e68a3..03a6c0878de43 100644 --- a/src/connector/src/source/filesystem/s3_v2/lister.rs +++ b/src/connector/src/source/filesystem/s3_v2/lister.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; +use anyhow::Context; use async_trait::async_trait; -use aws_sdk_s3::error::DisplayErrorContext; use aws_sdk_s3::types::Object; use itertools::Itertools; +use crate::error::ConnectorResult; use crate::source::filesystem::{FsPageItem, S3SplitEnumerator}; use crate::source::{FsFilterCtrlCtx, FsListInner}; @@ -25,7 +25,7 @@ use crate::source::{FsFilterCtrlCtx, FsListInner}; impl FsListInner for S3SplitEnumerator { async fn get_next_page From<&'a Object>>( &mut self, - ) -> anyhow::Result<(Vec, bool)> { + ) -> ConnectorResult<(Vec, bool)> { let mut has_finished = false; let mut req = self .client @@ -38,7 +38,7 @@ impl FsListInner for S3SplitEnumerator { let mut res = req .send() .await - .map_err(|e| anyhow!(DisplayErrorContext(e)))?; + .with_context(|| format!("failed to list objects in bucket `{}`", self.bucket_name))?; if res.is_truncated().unwrap_or_default() { self.next_continuation_token = res.next_continuation_token.clone(); } else { diff --git a/src/connector/src/source/google_pubsub/enumerator/client.rs b/src/connector/src/source/google_pubsub/enumerator/client.rs index 01809a3c773b0..25cb28909c479 100644 --- a/src/connector/src/source/google_pubsub/enumerator/client.rs +++ b/src/connector/src/source/google_pubsub/enumerator/client.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{anyhow, bail}; +use anyhow::Context; use async_trait::async_trait; use chrono::{TimeZone, Utc}; use google_cloud_pubsub::client::{Client, ClientConfig}; use google_cloud_pubsub::subscription::{SeekTo, SubscriptionConfig}; +use risingwave_common::bail; +use crate::error::ConnectorResult; use crate::source::base::SplitEnumerator; use crate::source::google_pubsub::split::PubsubSplit; use crate::source::google_pubsub::PubsubProperties; @@ -36,7 +38,7 @@ impl SplitEnumerator for PubsubSplitEnumerator { async fn new( properties: Self::Properties, _context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> ConnectorResult { let subscription = properties.subscription.to_owned(); if properties.credentials.is_none() && properties.emulator_host.is_none() { @@ -49,20 +51,23 @@ impl SplitEnumerator for PubsubSplitEnumerator { let config = ClientConfig::default().with_auth().await?; let client = Client::new(config) .await - .map_err(|e| anyhow!("error initializing pubsub client: {:?}", e))?; + .context("error initializing pubsub client")?; let sub = client.subscription(&subscription); if !sub .exists(None) .await - .map_err(|e| anyhow!("error checking subscription validity: {:?}", e))? + .context("error checking subscription validity")? { bail!("subscription {} does not exist", &subscription) } // We need the `retain_acked_messages` configuration to be true to seek back to timestamps // as done in the [`PubsubSplitReader`] and here. - let (_, subscription_config) = sub.config(None).await?; + let (_, subscription_config) = sub + .config(None) + .await + .context("failed to fetch subscription config")?; if let SubscriptionConfig { retain_acked_messages: false, .. @@ -76,7 +81,7 @@ impl SplitEnumerator for PubsubSplitEnumerator { (Some(start_offset), None) => { let ts = start_offset .parse::() - .map_err(|e| anyhow!("error parsing start_offset: {:?}", e)) + .context("error parsing start_offset") .map(|nanos| Utc.timestamp_nanos(nanos).into())?; Some(SeekTo::Timestamp(ts)) } @@ -89,7 +94,7 @@ impl SplitEnumerator for PubsubSplitEnumerator { if let Some(seek_to) = seek_to { sub.seek(seek_to, None) .await - .map_err(|e| anyhow!("error seeking subscription: {:?}", e))?; + .context("error seeking subscription")?; } Ok(Self { @@ -98,7 +103,7 @@ impl SplitEnumerator for PubsubSplitEnumerator { }) } - async fn list_splits(&mut self) -> anyhow::Result> { + async fn list_splits(&mut self) -> ConnectorResult> { tracing::debug!("enumerating pubsub splits"); let splits: Vec = (0..self.split_count) .map(|i| PubsubSplit { diff --git a/src/connector/src/source/google_pubsub/mod.rs b/src/connector/src/source/google_pubsub/mod.rs index aeec1accd820b..0a49fa6467f66 100644 --- a/src/connector/src/source/google_pubsub/mod.rs +++ b/src/connector/src/source/google_pubsub/mod.rs @@ -101,9 +101,8 @@ impl PubsubProperties { #[cfg(test)] mod tests { - use anyhow::Result; - use super::*; + use crate::error::ConnectorResult as Result; const EMULATOR_HOST: &str = "localhost:8081"; const CREDENTIALS: &str = "{}"; diff --git a/src/connector/src/source/google_pubsub/source/reader.rs b/src/connector/src/source/google_pubsub/source/reader.rs index dfaf21ad3164e..0887cb06594f9 100644 --- a/src/connector/src/source/google_pubsub/source/reader.rs +++ b/src/connector/src/source/google_pubsub/source/reader.rs @@ -12,20 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{anyhow, ensure, Context, Result}; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use chrono::{NaiveDateTime, TimeZone, Utc}; use futures_async_stream::try_stream; use google_cloud_pubsub::client::{Client, ClientConfig}; use google_cloud_pubsub::subscription::{SeekTo, Subscription}; -use risingwave_common::bail; +use risingwave_common::{bail, ensure}; use tonic::Code; use super::TaggedReceivedMessage; +use crate::error::{ConnectorError, ConnectorResult as Result}; use crate::parser::ParserConfig; use crate::source::google_pubsub::{PubsubProperties, PubsubSplit}; use crate::source::{ - into_chunk_stream, BoxSourceWithStateStream, Column, CommonSplitReader, SourceContextRef, + into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef, SourceMessage, SplitId, SplitMetaData, SplitReader, }; @@ -41,7 +42,7 @@ pub struct PubsubSplitReader { } impl CommonSplitReader for PubsubSplitReader { - #[try_stream(ok = Vec, error = anyhow::Error)] + #[try_stream(ok = Vec, error = ConnectorError)] async fn into_data_stream(self) { loop { let pull_result = self @@ -135,12 +136,12 @@ impl SplitReader for PubsubSplitReader { .as_str() .parse::() .map(|nanos| Utc.timestamp_nanos(nanos)) - .map_err(|e| anyhow!("error parsing offset: {:?}", e))?; + .context("error parsing offset")?; subscription .seek(SeekTo::Timestamp(timestamp.into()), None) .await - .map_err(|e| anyhow!("error seeking to pubsub offset: {:?}", e))?; + .context("error seeking to pubsub offset")?; } let stop_offset = if let Some(ref offset) = split.stop_offset { @@ -164,7 +165,7 @@ impl SplitReader for PubsubSplitReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { let parser_config = self.parser_config.clone(); let source_context = self.source_ctx.clone(); into_chunk_stream(self, parser_config, source_context) diff --git a/src/connector/src/source/google_pubsub/split.rs b/src/connector/src/source/google_pubsub/split.rs index e52ffa8ef72a4..f150f7f08038b 100644 --- a/src/connector/src/source/google_pubsub/split.rs +++ b/src/connector/src/source/google_pubsub/split.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::{SplitId, SplitMetaData}; #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)] @@ -36,8 +36,8 @@ pub struct PubsubSplit { } impl SplitMetaData for PubsubSplit { - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { @@ -48,7 +48,7 @@ impl SplitMetaData for PubsubSplit { format!("{}-{}", self.subscription, self.index).into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { self.start_offset = Some(start_offset); Ok(()) } diff --git a/src/connector/src/source/iceberg/mod.rs b/src/connector/src/source/iceberg/mod.rs new file mode 100644 index 0000000000000..776ab65a05540 --- /dev/null +++ b/src/connector/src/source/iceberg/mod.rs @@ -0,0 +1,204 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; + +use anyhow::anyhow; +use async_trait::async_trait; +use icelake::types::DataContentType; +use itertools::Itertools; +use risingwave_common::bail; +use risingwave_common::types::JsonbVal; +use serde::{Deserialize, Serialize}; + +use crate::error::ConnectorResult; +use crate::parser::ParserConfig; +use crate::sink::iceberg::IcebergConfig; +use crate::source::{ + BoxChunkSourceStream, Column, SourceContextRef, SourceEnumeratorContextRef, SourceProperties, + SplitEnumerator, SplitId, SplitMetaData, SplitReader, UnknownFields, +}; + +pub const ICEBERG_CONNECTOR: &str = "iceberg"; + +#[derive(Clone, Debug, Deserialize, PartialEq, with_options::WithOptions)] +pub struct IcebergProperties { + #[serde(rename = "catalog.type")] + pub catalog_type: String, + #[serde(rename = "s3.region")] + pub region_name: String, + #[serde(rename = "s3.endpoint", default)] + pub endpoint: String, + #[serde(rename = "s3.access.key", default)] + pub s3_access: String, + #[serde(rename = "s3.secret.key", default)] + pub s3_secret: String, + #[serde(rename = "warehouse.path")] + pub warehouse_path: String, + #[serde(rename = "database.name")] + pub database_name: String, + #[serde(rename = "table.name")] + pub table_name: String, + + #[serde(flatten)] + pub unknown_fields: HashMap, +} + +impl IcebergProperties { + pub fn to_iceberg_config(&self) -> IcebergConfig { + IcebergConfig { + database_name: Some(self.database_name.clone()), + table_name: self.table_name.clone(), + catalog_type: Some(self.catalog_type.clone()), + path: self.warehouse_path.clone(), + endpoint: Some(self.endpoint.clone()), + access_key: self.s3_access.clone(), + secret_key: self.s3_secret.clone(), + region: Some(self.region_name.clone()), + ..Default::default() + } + } +} + +impl SourceProperties for IcebergProperties { + type Split = IcebergSplit; + type SplitEnumerator = IcebergSplitEnumerator; + type SplitReader = IcebergFileReader; + + const SOURCE_NAME: &'static str = ICEBERG_CONNECTOR; +} + +impl UnknownFields for IcebergProperties { + fn unknown_fields(&self) -> HashMap { + self.unknown_fields.clone() + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize)] +pub struct IcebergSplit { + pub split_id: i64, + pub snapshot_id: i64, + pub files: Vec, +} + +impl SplitMetaData for IcebergSplit { + fn id(&self) -> SplitId { + self.split_id.to_string().into() + } + + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(|e| anyhow!(e).into()) + } + + fn encode_to_json(&self) -> JsonbVal { + serde_json::to_value(self.clone()).unwrap().into() + } + + fn update_with_offset(&mut self, _start_offset: String) -> ConnectorResult<()> { + unimplemented!() + } +} + +#[derive(Debug, Clone)] +pub struct IcebergSplitEnumerator { + config: IcebergConfig, +} + +#[async_trait] +impl SplitEnumerator for IcebergSplitEnumerator { + type Properties = IcebergProperties; + type Split = IcebergSplit; + + async fn new( + properties: Self::Properties, + _context: SourceEnumeratorContextRef, + ) -> ConnectorResult { + let iceberg_config = properties.to_iceberg_config(); + Ok(Self { + config: iceberg_config, + }) + } + + async fn list_splits(&mut self) -> ConnectorResult> { + // Iceberg source does not support streaming queries + Ok(vec![]) + } +} + +impl IcebergSplitEnumerator { + pub async fn list_splits_batch( + &self, + batch_parallelism: usize, + ) -> ConnectorResult> { + if batch_parallelism == 0 { + bail!("Batch parallelism is 0. Cannot split the iceberg files."); + } + let table = self.config.load_table().await?; + let snapshot_id = table.current_table_metadata().current_snapshot_id.unwrap(); + let mut files = vec![]; + for file in table.current_data_files().await? { + if file.content != DataContentType::Data { + bail!("Reading iceberg table with delete files is unsupported. Please try to compact the table first."); + } + files.push(file.file_path); + } + let split_num = batch_parallelism; + // evenly split the files into splits based on the parallelism. + let split_size = files.len() / split_num; + let remaining = files.len() % split_num; + let mut splits = vec![]; + for i in 0..split_num { + let start = i * split_size; + let end = (i + 1) * split_size; + let split = IcebergSplit { + split_id: i as i64, + snapshot_id, + files: files[start..end].to_vec(), + }; + splits.push(split); + } + for i in 0..remaining { + splits[i] + .files + .push(files[split_num * split_size + i].clone()); + } + Ok(splits + .into_iter() + .filter(|split| !split.files.is_empty()) + .collect_vec()) + } +} + +#[derive(Debug)] +pub struct IcebergFileReader {} + +#[async_trait] +impl SplitReader for IcebergFileReader { + type Properties = IcebergProperties; + type Split = IcebergSplit; + + async fn new( + _props: IcebergProperties, + _splits: Vec, + _parser_config: ParserConfig, + _source_ctx: SourceContextRef, + _columns: Option>, + ) -> ConnectorResult { + unimplemented!() + } + + fn into_stream(self) -> BoxChunkSourceStream { + unimplemented!() + } +} diff --git a/src/connector/src/source/kafka/enumerator/client.rs b/src/connector/src/source/kafka/enumerator/client.rs index b0ae57f610492..4441be8c9db21 100644 --- a/src/connector/src/source/kafka/enumerator/client.rs +++ b/src/connector/src/source/kafka/enumerator/client.rs @@ -15,12 +15,14 @@ use std::collections::HashMap; use std::time::Duration; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use rdkafka::consumer::{BaseConsumer, Consumer}; use rdkafka::error::KafkaResult; use rdkafka::{Offset, TopicPartitionList}; +use risingwave_common::bail; +use crate::error::ConnectorResult; use crate::source::base::SplitEnumerator; use crate::source::kafka::split::KafkaSplit; use crate::source::kafka::{KafkaProperties, PrivateLinkConsumerContext, KAFKA_ISOLATION_LEVEL}; @@ -57,12 +59,12 @@ impl SplitEnumerator for KafkaSplitEnumerator { async fn new( properties: KafkaProperties, context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> ConnectorResult { let mut config = rdkafka::ClientConfig::new(); let common_props = &properties.common; let broker_address = common_props.brokers.clone(); - let broker_rewrite_map = common_props.broker_rewrite_map.clone(); + let broker_rewrite_map = properties.privatelink_common.broker_rewrite_map.clone(); let topic = common_props.topic.clone(); config.set("bootstrap.servers", &broker_address); config.set("isolation.level", KAFKA_ISOLATION_LEVEL); @@ -77,11 +79,9 @@ impl SplitEnumerator for KafkaSplitEnumerator { Some("earliest") => KafkaEnumeratorOffset::Earliest, Some("latest") => KafkaEnumeratorOffset::Latest, None => KafkaEnumeratorOffset::Earliest, - _ => { - return Err(anyhow!( - "properties `scan_startup_mode` only support earliest and latest or leave it empty" - )); - } + _ => bail!( + "properties `scan_startup_mode` only support earliest and latest or leave it empty" + ), }; if let Some(s) = &properties.time_offset { @@ -105,13 +105,14 @@ impl SplitEnumerator for KafkaSplitEnumerator { }) } - async fn list_splits(&mut self) -> anyhow::Result> { - let topic_partitions = self.fetch_topic_partition().await.map_err(|e| { - anyhow!(format!( - "failed to fetch metadata from kafka ({}), error: {}", - self.broker_address, e - )) + async fn list_splits(&mut self) -> ConnectorResult> { + let topic_partitions = self.fetch_topic_partition().await.with_context(|| { + format!( + "failed to fetch metadata from kafka ({})", + self.broker_address + ) })?; + let watermarks = self.get_watermarks(topic_partitions.as_ref()).await?; let mut start_offsets = self .fetch_start_offset(topic_partitions.as_ref(), &watermarks) @@ -153,12 +154,12 @@ impl KafkaSplitEnumerator { &mut self, expect_start_timestamp_millis: Option, expect_stop_timestamp_millis: Option, - ) -> anyhow::Result> { - let topic_partitions = self.fetch_topic_partition().await.map_err(|e| { - anyhow!(format!( - "failed to fetch metadata from kafka ({}), error: {}", - self.broker_address, e - )) + ) -> ConnectorResult> { + let topic_partitions = self.fetch_topic_partition().await.with_context(|| { + format!( + "failed to fetch metadata from kafka ({})", + self.broker_address + ) })?; // here we are getting the start offset and end offset for each partition with the given @@ -349,7 +350,7 @@ impl KafkaSplitEnumerator { .is_ok() } - async fn fetch_topic_partition(&self) -> anyhow::Result> { + async fn fetch_topic_partition(&self) -> ConnectorResult> { // for now, we only support one topic let metadata = self .client @@ -358,11 +359,11 @@ impl KafkaSplitEnumerator { let topic_meta = match metadata.topics() { [meta] => meta, - _ => return Err(anyhow!("topic {} not found", self.topic)), + _ => bail!("topic {} not found", self.topic), }; if topic_meta.partitions().is_empty() { - return Err(anyhow!("topic {} not found", self.topic)); + bail!("topic {} not found", self.topic); } Ok(topic_meta diff --git a/src/connector/src/source/kafka/mod.rs b/src/connector/src/source/kafka/mod.rs index d2879302f7ece..52d410a8ee717 100644 --- a/src/connector/src/source/kafka/mod.rs +++ b/src/connector/src/source/kafka/mod.rs @@ -17,6 +17,8 @@ use std::collections::HashMap; use serde::Deserialize; use serde_with::{serde_as, DisplayFromStr}; +use crate::common::KafkaPrivateLinkCommon; + pub mod enumerator; pub mod private_link; pub mod source; @@ -130,6 +132,9 @@ pub struct KafkaProperties { #[serde(flatten)] pub rdkafka_properties_consumer: RdKafkaPropertiesConsumer, + #[serde(flatten)] + pub privatelink_common: KafkaPrivateLinkCommon, + #[serde(flatten)] pub unknown_fields: HashMap, } @@ -208,6 +213,8 @@ mod test { "properties.fetch.max.bytes".to_string() => "114514".to_string(), "properties.enable.auto.commit".to_string() => "true".to_string(), "properties.fetch.queue.backoff.ms".to_string() => "114514".to_string(), + // PrivateLink + "broker.rewrite.endpoints".to_string() => "{\"broker1\": \"10.0.0.1:8001\"}".to_string(), }; let props: KafkaProperties = @@ -246,5 +253,9 @@ mod test { props.rdkafka_properties_consumer.fetch_queue_backoff_ms, Some(114514) ); + let hashmap: HashMap = hashmap! { + "broker1".to_string() => "10.0.0.1:8001".to_string() + }; + assert_eq!(props.privatelink_common.broker_rewrite_map, Some(hashmap)); } } diff --git a/src/connector/src/source/kafka/private_link.rs b/src/connector/src/source/kafka/private_link.rs index 5ba99a87d414e..3eebacca09f93 100644 --- a/src/connector/src/source/kafka/private_link.rs +++ b/src/connector/src/source/kafka/private_link.rs @@ -16,17 +16,21 @@ use std::collections::{BTreeMap, HashMap}; use std::str::FromStr; use std::sync::Arc; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use itertools::Itertools; use rdkafka::client::BrokerAddr; use rdkafka::consumer::ConsumerContext; use rdkafka::producer::{DeliveryResult, ProducerContext}; use rdkafka::{ClientContext, Statistics}; +use risingwave_common::bail; use risingwave_common::util::addr::HostAddr; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_pb::catalog::connection::PrivateLinkService; -use crate::common::{AwsPrivateLinkItem, BROKER_REWRITE_MAP_KEY, PRIVATE_LINK_TARGETS_KEY}; +use crate::common::{ + AwsPrivateLinkItem, PRIVATE_LINK_BROKER_REWRITE_MAP_KEY, PRIVATE_LINK_TARGETS_KEY, +}; +use crate::error::ConnectorResult; use crate::source::kafka::stats::RdKafkaStats; use crate::source::kafka::{KAFKA_PROPS_BROKER_KEY, KAFKA_PROPS_BROKER_KEY_ALIAS}; use crate::source::KAFKA_CONNECTOR; @@ -66,9 +70,9 @@ impl BrokerAddrRewriter { pub fn new( role: PrivateLinkContextRole, broker_rewrite_map: Option>, - ) -> anyhow::Result { + ) -> ConnectorResult { tracing::info!("[{}] rewrite map {:?}", role, broker_rewrite_map); - let rewrite_map: anyhow::Result> = broker_rewrite_map + let rewrite_map: ConnectorResult> = broker_rewrite_map .map_or(Ok(BTreeMap::new()), |addr_map| { addr_map .into_iter() @@ -107,7 +111,7 @@ impl PrivateLinkConsumerContext { broker_rewrite_map: Option>, identifier: Option, metrics: Option>, - ) -> anyhow::Result { + ) -> ConnectorResult { let inner = BrokerAddrRewriter::new(PrivateLinkContextRole::Consumer, broker_rewrite_map)?; Ok(Self { inner, @@ -150,7 +154,7 @@ impl PrivateLinkProducerContext { broker_rewrite_map: Option>, identifier: Option, metrics: Option>, - ) -> anyhow::Result { + ) -> ConnectorResult { let inner = BrokerAddrRewriter::new(PrivateLinkContextRole::Producer, broker_rewrite_map)?; Ok(Self { inner, @@ -193,11 +197,12 @@ fn kafka_props_broker_key(with_properties: &BTreeMap) -> &str { fn get_property_required( with_properties: &BTreeMap, property: &str, -) -> anyhow::Result { +) -> ConnectorResult { with_properties .get(property) .map(|s| s.to_lowercase()) - .ok_or_else(|| anyhow!("Required property \"{property}\" is not provided")) + .with_context(|| format!("Required property \"{property}\" is not provided")) + .map_err(Into::into) } #[inline(always)] @@ -211,23 +216,25 @@ fn is_kafka_connector(with_properties: &BTreeMap) -> bool { } pub fn insert_privatelink_broker_rewrite_map( - properties: &mut BTreeMap, + with_options: &mut BTreeMap, svc: Option<&PrivateLinkService>, privatelink_endpoint: Option, -) -> anyhow::Result<()> { +) -> ConnectorResult<()> { let mut broker_rewrite_map = HashMap::new(); - let servers = get_property_required(properties, kafka_props_broker_key(properties))?; + let servers = get_property_required(with_options, kafka_props_broker_key(with_options))?; let broker_addrs = servers.split(',').collect_vec(); - let link_target_value = get_property_required(properties, PRIVATE_LINK_TARGETS_KEY)?; + let link_target_value = get_property_required(with_options, PRIVATE_LINK_TARGETS_KEY)?; let link_targets: Vec = serde_json::from_str(link_target_value.as_str()).map_err(|e| anyhow!(e))?; + // remove the private link targets from WITH options, as they are useless after we constructed the rewrite mapping + with_options.remove(PRIVATE_LINK_TARGETS_KEY); if broker_addrs.len() != link_targets.len() { - return Err(anyhow!( + bail!( "The number of broker addrs {} does not match the number of private link targets {}", broker_addrs.len(), link_targets.len() - )); + ); } if let Some(endpoint) = privatelink_endpoint { @@ -237,15 +244,15 @@ pub fn insert_privatelink_broker_rewrite_map( } } else { if svc.is_none() { - return Err(anyhow!("Privatelink endpoint not found.",)); + bail!("Privatelink endpoint not found."); } let svc = svc.unwrap(); for (link, broker) in link_targets.iter().zip_eq_fast(broker_addrs.into_iter()) { if svc.dns_entries.is_empty() { - return Err(anyhow!( + bail!( "No available private link endpoints for Kafka broker {}", broker - )); + ); } // rewrite the broker address to the dns name w/o az // requires the NLB has enabled the cross-zone load balancing @@ -259,6 +266,6 @@ pub fn insert_privatelink_broker_rewrite_map( // save private link dns names into source properties, which // will be extracted into KafkaProperties let json = serde_json::to_string(&broker_rewrite_map).map_err(|e| anyhow!(e))?; - properties.insert(BROKER_REWRITE_MAP_KEY.to_string(), json); + with_options.insert(PRIVATE_LINK_BROKER_REWRITE_MAP_KEY.to_string(), json); Ok(()) } diff --git a/src/connector/src/source/kafka/source/message.rs b/src/connector/src/source/kafka/source/message.rs index 52f9722533136..0ef55dc79132d 100644 --- a/src/connector/src/source/kafka/source/message.rs +++ b/src/connector/src/source/kafka/source/message.rs @@ -16,6 +16,8 @@ use itertools::Itertools; use rdkafka::message::{BorrowedMessage, Headers, OwnedHeaders}; use rdkafka::Message; use risingwave_common::types::{Datum, ListValue, Scalar, ScalarImpl, StructValue}; +use risingwave_pb::data::data_type::TypeName as PbTypeName; +use risingwave_pb::data::DataType as PbDataType; use crate::parser::additional_columns::get_kafka_header_item_datatype; use crate::source::base::SourceMessage; @@ -39,6 +41,31 @@ impl KafkaMeta { .into() } + pub fn extract_header_inner( + &self, + inner_field: &str, + data_type: Option<&PbDataType>, + ) -> Option { + let target_value = self + .headers + .as_ref() + .iter() + .find_map(|headers| { + headers + .iter() + .find(|header| header.key == inner_field) + .map(|header| header.value) + }) + .unwrap_or(None); // if not found the specified column, return None + if let Some(data_type) = data_type + && data_type.type_name == PbTypeName::Varchar as i32 + { + Some(target_value.map(|byte| ScalarImpl::Utf8(String::from_utf8_lossy(byte).into()))) + } else { + Some(target_value.map(|byte| ScalarImpl::Bytea(byte.into()))) + } + } + pub fn extract_headers(&self) -> Option { self.headers.as_ref().map(|headers| { let header_item: Vec = headers diff --git a/src/connector/src/source/kafka/source/reader.rs b/src/connector/src/source/kafka/source/reader.rs index 7f0cab97122ae..d67e45fee3837 100644 --- a/src/connector/src/source/kafka/source/reader.rs +++ b/src/connector/src/source/kafka/source/reader.rs @@ -17,7 +17,7 @@ use std::collections::HashMap; use std::mem::swap; use std::time::Duration; -use anyhow::{anyhow, Result}; +use anyhow::Context; use async_trait::async_trait; use futures::StreamExt; use futures_async_stream::try_stream; @@ -25,16 +25,17 @@ use rdkafka::config::RDKafkaLogLevel; use rdkafka::consumer::{Consumer, StreamConsumer}; use rdkafka::error::KafkaError; use rdkafka::{ClientConfig, Message, Offset, TopicPartitionList}; -use risingwave_pb::plan_common::AdditionalColumnType; +use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; +use crate::error::ConnectorResult as Result; use crate::parser::ParserConfig; use crate::source::base::SourceMessage; use crate::source::kafka::{ KafkaProperties, KafkaSplit, PrivateLinkConsumerContext, KAFKA_ISOLATION_LEVEL, }; use crate::source::{ - into_chunk_stream, BoxSourceWithStateStream, Column, CommonSplitReader, SourceContextRef, - SplitId, SplitMetaData, SplitReader, + into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef, SplitId, + SplitMetaData, SplitReader, }; pub struct KafkaSplitReader { @@ -61,7 +62,7 @@ impl SplitReader for KafkaSplitReader { let mut config = ClientConfig::new(); let bootstrap_servers = &properties.common.brokers; - let broker_rewrite_map = properties.common.broker_rewrite_map.clone(); + let broker_rewrite_map = properties.privatelink_common.broker_rewrite_map.clone(); // disable partition eof config.set("enable.partition.eof", "false"); @@ -98,7 +99,7 @@ impl SplitReader for KafkaSplitReader { .set_log_level(RDKafkaLogLevel::Info) .create_with_context(client_ctx) .await - .map_err(|e| anyhow!("failed to create kafka consumer: {}", e))?; + .context("failed to create kafka consumer")?; let mut tpl = TopicPartitionList::with_capacity(splits.len()); @@ -145,7 +146,7 @@ impl SplitReader for KafkaSplitReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { let parser_config = self.parser_config.clone(); let source_context = self.source_ctx.clone(); into_chunk_stream(self, parser_config, source_context) @@ -168,7 +169,7 @@ impl KafkaSplitReader { } impl CommonSplitReader for KafkaSplitReader { - #[try_stream(ok = Vec, error = anyhow::Error)] + #[try_stream(ok = Vec, error = crate::error::ConnectorError)] async fn into_data_stream(self) { if self.offsets.values().all(|(start_offset, stop_offset)| { match (start_offset, stop_offset) { @@ -196,12 +197,12 @@ impl CommonSplitReader for KafkaSplitReader { let max_chunk_size = self.source_ctx.source_ctrl_opts.chunk_size; let mut res = Vec::with_capacity(max_chunk_size); // ingest kafka message header can be expensive, do it only when required - let require_message_header = self - .parser_config - .common - .rw_columns - .iter() - .any(|col_desc| col_desc.additional_column_type == AdditionalColumnType::Header); + let require_message_header = self.parser_config.common.rw_columns.iter().any(|col_desc| { + matches!( + col_desc.additional_column.column_type, + Some(AdditionalColumnType::Headers(_) | AdditionalColumnType::HeaderInner(_)) + ) + }); #[for_await] 'for_outer_loop: for msgs in self.consumer.stream().ready_chunks(max_chunk_size) { diff --git a/src/connector/src/source/kafka/split.rs b/src/connector/src/source/kafka/split.rs index 1043cdd01f2f6..a98707eb4ab3a 100644 --- a/src/connector/src/source/kafka/split.rs +++ b/src/connector/src/source/kafka/split.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::{SplitId, SplitMetaData}; #[derive(Clone, Serialize, Deserialize, Debug, PartialEq, Hash)] @@ -32,15 +32,15 @@ impl SplitMetaData for KafkaSplit { format!("{}", self.partition).into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { self.start_offset = Some(start_offset.as_str().parse::().unwrap()); Ok(()) } diff --git a/src/connector/src/source/kinesis/enumerator/client.rs b/src/connector/src/source/kinesis/enumerator/client.rs index bbf4e29258260..b8966f99ae1ac 100644 --- a/src/connector/src/source/kinesis/enumerator/client.rs +++ b/src/connector/src/source/kinesis/enumerator/client.rs @@ -12,11 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::Result; +use anyhow::Context as _; use async_trait::async_trait; use aws_sdk_kinesis::types::Shard; use aws_sdk_kinesis::Client as kinesis_client; +use risingwave_common::bail; +use crate::error::ConnectorResult as Result; use crate::source::kinesis::split::{KinesisOffset, KinesisSplit}; use crate::source::kinesis::*; use crate::source::{SourceEnumeratorContextRef, SplitEnumerator}; @@ -56,15 +58,11 @@ impl SplitEnumerator for KinesisSplitEnumerator { .set_next_token(next_token) .stream_name(&self.stream_name) .send() - .await?; + .await + .context("failed to list kinesis shards")?; match list_shard_output.shards { Some(shard) => shard_collect.extend(shard), - None => { - return Err(anyhow::Error::msg(format!( - "no shards in stream {}", - &self.stream_name - ))); - } + None => bail!("no shards in stream {}", &self.stream_name), } match list_shard_output.next_token { diff --git a/src/connector/src/source/kinesis/source/reader.rs b/src/connector/src/source/kinesis/source/reader.rs index 0e5b07d97736a..8b8b2f104cec2 100644 --- a/src/connector/src/source/kinesis/source/reader.rs +++ b/src/connector/src/source/kinesis/source/reader.rs @@ -14,22 +14,25 @@ use std::time::Duration; -use anyhow::{anyhow, Result}; +use anyhow::{anyhow, Context}; use async_trait::async_trait; -use aws_sdk_kinesis::error::{DisplayErrorContext, SdkError}; +use aws_sdk_kinesis::error::SdkError; use aws_sdk_kinesis::operation::get_records::{GetRecordsError, GetRecordsOutput}; use aws_sdk_kinesis::primitives::DateTime; use aws_sdk_kinesis::types::ShardIteratorType; use aws_sdk_kinesis::Client as KinesisClient; use futures_async_stream::try_stream; +use risingwave_common::bail; +use thiserror_ext::AsReport; use tokio_retry; +use crate::error::ConnectorResult as Result; use crate::parser::ParserConfig; use crate::source::kinesis::source::message::from_kinesis_record; use crate::source::kinesis::split::{KinesisOffset, KinesisSplit}; use crate::source::kinesis::KinesisProperties; use crate::source::{ - into_chunk_stream, BoxSourceWithStateStream, Column, CommonSplitReader, SourceContextRef, + into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef, SourceMessage, SplitId, SplitMetaData, SplitReader, }; @@ -74,13 +77,11 @@ impl SplitReader for KinesisSplitReader { if let Some(ts) = &properties.timestamp_offset { KinesisOffset::Timestamp(*ts) } else { - return Err(anyhow!("scan.startup.timestamp.millis is required")); + bail!("scan.startup.timestamp.millis is required"); } } _ => { - return Err(anyhow!( - "invalid scan_startup_mode, accept earliest/latest/timestamp" - )) + bail!("invalid scan_startup_mode, accept earliest/latest/timestamp") } }, }, @@ -90,9 +91,7 @@ impl SplitReader for KinesisSplitReader { if !matches!(start_position, KinesisOffset::Timestamp(_)) && properties.timestamp_offset.is_some() { - return Err( - anyhow!("scan.startup.mode need to be set to 'timestamp' if you want to start with a specific timestamp") - ); + bail!("scan.startup.mode need to be set to 'timestamp' if you want to start with a specific timestamp"); } let stream_name = properties.common.stream_name.clone(); @@ -113,7 +112,7 @@ impl SplitReader for KinesisSplitReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { let parser_config = self.parser_config.clone(); let source_context = self.source_ctx.clone(); into_chunk_stream(self, parser_config, source_context) @@ -121,7 +120,7 @@ impl SplitReader for KinesisSplitReader { } impl CommonSplitReader for KinesisSplitReader { - #[try_stream(ok = Vec < SourceMessage >, error = anyhow::Error)] + #[try_stream(ok = Vec < SourceMessage >, error = crate::error::ConnectorError)] async fn into_data_stream(mut self) { self.new_shard_iter().await?; loop { @@ -189,14 +188,12 @@ impl CommonSplitReader for KinesisSplitReader { continue; } Err(e) => { - let error_msg = format!( - "Kinesis got a unhandled error: {:?}, stream {:?}, shard {:?}", - DisplayErrorContext(e), - self.stream_name, - self.shard_id, - ); - tracing::error!("{}", error_msg); - return Err(anyhow!("{}", error_msg)); + let error = anyhow!(e).context(format!( + "Kinesis got a unhandled error on stream {:?}, shard {:?}", + self.stream_name, self.shard_id + )); + tracing::error!(error = %error.as_report()); + return Err(error.into()); } } } @@ -246,12 +243,12 @@ impl KinesisSplitReader { .set_timestamp(starting_timestamp) .send() .await - .map_err(|e| anyhow!(DisplayErrorContext(e)))?; + .context("failed to get kinesis shard iterator")?; if let Some(iter) = resp.shard_iterator() { Ok(iter.to_owned()) } else { - Err(anyhow!("shard iterator is none")) + bail!("shard iterator is none") } } diff --git a/src/connector/src/source/kinesis/split.rs b/src/connector/src/source/kinesis/split.rs index c327e6da61e92..1c7bea61f8744 100644 --- a/src/connector/src/source/kinesis/split.rs +++ b/src/connector/src/source/kinesis/split.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::{SplitId, SplitMetaData}; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, Hash)] @@ -39,15 +39,15 @@ impl SplitMetaData for KinesisSplit { self.shard_id.clone() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { let start_offset = if start_offset.is_empty() { KinesisOffset::Earliest } else { diff --git a/src/connector/src/source/manager.rs b/src/connector/src/source/manager.rs index 07ce5a3d4898e..a5584f6af83d5 100644 --- a/src/connector/src/source/manager.rs +++ b/src/connector/src/source/manager.rs @@ -19,7 +19,7 @@ use risingwave_common::catalog::{ TABLE_NAME_COLUMN_NAME, }; use risingwave_common::types::DataType; -use risingwave_pb::plan_common::{AdditionalColumnType, ColumnDescVersion}; +use risingwave_pb::plan_common::{AdditionalColumn, ColumnDescVersion}; /// `SourceColumnDesc` is used to describe a column in the Source and is used as the column /// counterpart in `StreamScan` @@ -31,13 +31,16 @@ pub struct SourceColumnDesc { pub fields: Vec, pub column_type: SourceColumnType, - // `is_pk` is used to indicate whether the column is part of the primary key columns. + /// `is_pk` is used to indicate whether the column is part of the primary key columns. pub is_pk: bool, - // `additional_column_type` and `column_type` are orthogonal - // `additional_column_type` is used to indicate the column is from which part of the message - // `column_type` is used to indicate the type of the column, only used in cdc scenario - pub additional_column_type: AdditionalColumnType, + /// `is_hidden_addition_col` is used to indicate whether the column is a hidden addition column. + pub is_hidden_addition_col: bool, + + /// `additional_column` and `column_type` are orthogonal + /// `additional_column` is used to indicate the column is from which part of the message + /// `column_type` is used to indicate the type of the column, only used in cdc scenario + pub additional_column: AdditionalColumn, } /// `SourceColumnType` is used to indicate the type of a column emitted by the Source. @@ -87,7 +90,15 @@ impl SourceColumnDesc { fields: vec![], column_type: SourceColumnType::Normal, is_pk: false, - additional_column_type: AdditionalColumnType::Normal, + is_hidden_addition_col: false, + additional_column: AdditionalColumn { column_type: None }, + } + } + + pub fn hidden_addition_col_from_column_desc(c: &ColumnDesc) -> Self { + Self { + is_hidden_addition_col: true, + ..c.into() } } @@ -105,7 +116,7 @@ impl SourceColumnDesc { #[inline] pub fn is_visible(&self) -> bool { - self.column_type == SourceColumnType::Normal + !self.is_hidden_addition_col && self.column_type == SourceColumnType::Normal } } @@ -119,7 +130,8 @@ impl From<&ColumnDesc> for SourceColumnDesc { fields: c.field_descs.clone(), column_type, is_pk: false, - additional_column_type: c.additional_column_type, + is_hidden_addition_col: false, + additional_column: c.additional_column.clone(), } } } @@ -134,7 +146,7 @@ impl From<&SourceColumnDesc> for ColumnDesc { type_name: "".to_string(), generated_or_default_column: None, description: None, - additional_column_type: s.additional_column_type, + additional_column: s.additional_column.clone(), version: ColumnDescVersion::Pr13707, } } diff --git a/src/connector/src/source/mod.rs b/src/connector/src/source/mod.rs index 6cdc7d30e277a..3656820ed95b0 100644 --- a/src/connector/src/source/mod.rs +++ b/src/connector/src/source/mod.rs @@ -31,14 +31,13 @@ pub use kafka::KAFKA_CONNECTOR; pub use kinesis::KINESIS_CONNECTOR; pub use nats::NATS_CONNECTOR; mod common; +pub mod iceberg; mod manager; +pub mod reader; pub mod test_source; pub use manager::{SourceColumnDesc, SourceColumnType}; -pub use crate::parser::additional_columns::{ - get_connector_compatible_additional_columns, CompatibleAdditionalColumnsFn, -}; pub use crate::source::filesystem::opendal_source::{ GCS_CONNECTOR, OPENDAL_S3_CONNECTOR, POSIX_FS_CONNECTOR, }; diff --git a/src/connector/src/source/monitor/metrics.rs b/src/connector/src/source/monitor/metrics.rs index 5e47826b486da..bf43ca44212ae 100644 --- a/src/connector/src/source/monitor/metrics.rs +++ b/src/connector/src/source/monitor/metrics.rs @@ -79,14 +79,26 @@ impl SourceMetrics { let partition_input_count = register_int_counter_vec_with_registry!( "source_partition_input_count", "Total number of rows that have been input from specific partition", - &["actor_id", "source_id", "partition"], + &[ + "actor_id", + "source_id", + "partition", + "source_name", + "fragment_id" + ], registry ) .unwrap(); let partition_input_bytes = register_int_counter_vec_with_registry!( "source_partition_input_bytes", "Total bytes that have been input from specific partition", - &["actor_id", "source_id", "partition"], + &[ + "actor_id", + "source_id", + "partition", + "source_name", + "fragment_id" + ], registry ) .unwrap(); diff --git a/src/connector/src/source/nats/enumerator/mod.rs b/src/connector/src/source/nats/enumerator/mod.rs index c5059fdc8186c..557921747b8f0 100644 --- a/src/connector/src/source/nats/enumerator/mod.rs +++ b/src/connector/src/source/nats/enumerator/mod.rs @@ -14,11 +14,12 @@ use std::sync::Arc; -use anyhow; use async_trait::async_trait; +use risingwave_common::bail; use super::source::{NatsOffset, NatsSplit}; use super::NatsProperties; +use crate::error::ConnectorResult; use crate::source::{SourceEnumeratorContextRef, SplitEnumerator, SplitId}; #[derive(Debug, Clone)] @@ -36,7 +37,7 @@ impl SplitEnumerator for NatsSplitEnumerator { async fn new( properties: Self::Properties, _context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> ConnectorResult { let client = properties.common.build_client().await?; Ok(Self { subject: properties.common.subject, @@ -45,14 +46,14 @@ impl SplitEnumerator for NatsSplitEnumerator { }) } - async fn list_splits(&mut self) -> anyhow::Result> { + async fn list_splits(&mut self) -> ConnectorResult> { // Nats currently does not support list_splits API, if we simple return the default 0 without checking the client status, will result executor crash let state = self.client.connection_state(); if state != async_nats::connection::State::Connected { - return Err(anyhow::anyhow!( + bail!( "Nats connection status is not connected, current status is {:?}", state - )); + ); } // TODO: to simplify the logic, return 1 split for first version let nats_split = NatsSplit { diff --git a/src/connector/src/source/nats/source/reader.rs b/src/connector/src/source/nats/source/reader.rs index 88ba9d862a925..05954538b91cf 100644 --- a/src/connector/src/source/nats/source/reader.rs +++ b/src/connector/src/source/nats/source/reader.rs @@ -12,19 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{anyhow, Result}; +use anyhow::Context as _; use async_nats::jetstream::consumer; use async_trait::async_trait; use futures::StreamExt; use futures_async_stream::try_stream; +use risingwave_common::bail; use super::message::NatsMessage; use super::{NatsOffset, NatsSplit}; +use crate::error::ConnectorResult as Result; use crate::parser::ParserConfig; use crate::source::common::{into_chunk_stream, CommonSplitReader}; use crate::source::nats::NatsProperties; use crate::source::{ - BoxSourceWithStateStream, Column, SourceContextRef, SourceMessage, SplitId, SplitReader, + BoxChunkSourceStream, Column, SourceContextRef, SourceMessage, SplitId, SplitReader, }; pub struct NatsSplitReader { @@ -60,15 +62,15 @@ impl SplitReader for NatsSplitReader { "earliest" => NatsOffset::Earliest, "timestamp_millis" => { if let Some(time) = &properties.start_time { - NatsOffset::Timestamp(time.parse()?) + NatsOffset::Timestamp(time.parse().context( + "failed to parse the start time as nats offset timestamp", + )?) } else { - return Err(anyhow!("scan_startup_timestamp_millis is required")); + bail!("scan_startup_timestamp_millis is required"); } } _ => { - return Err(anyhow!( - "invalid scan_startup_mode, accept earliest/latest/timestamp_millis" - )) + bail!("invalid scan_startup_mode, accept earliest/latest/timestamp_millis") } }, }, @@ -93,7 +95,7 @@ impl SplitReader for NatsSplitReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { let parser_config = self.parser_config.clone(); let source_context = self.source_ctx.clone(); into_chunk_stream(self, parser_config, source_context) @@ -101,7 +103,7 @@ impl SplitReader for NatsSplitReader { } impl CommonSplitReader for NatsSplitReader { - #[try_stream(ok = Vec, error = anyhow::Error)] + #[try_stream(ok = Vec, error = crate::error::ConnectorError)] async fn into_data_stream(self) { let capacity = self.source_ctx.source_ctrl_opts.chunk_size; let messages = self.consumer.messages().await?; diff --git a/src/connector/src/source/nats/split.rs b/src/connector/src/source/nats/split.rs index 1a176102efb60..d3b4ded019016 100644 --- a/src/connector/src/source/nats/split.rs +++ b/src/connector/src/source/nats/split.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{anyhow, Ok}; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::{SplitId, SplitMetaData}; #[derive(Debug, Clone, Eq, PartialEq, Serialize, Deserialize, Hash)] @@ -43,15 +43,15 @@ impl SplitMetaData for NatsSplit { format!("{}", self.split_id).into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_sequence: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_sequence: String) -> ConnectorResult<()> { let start_sequence = if start_sequence.is_empty() { NatsOffset::Earliest } else { diff --git a/src/connector/src/source/nexmark/enumerator/mod.rs b/src/connector/src/source/nexmark/enumerator/mod.rs index b67a58f548965..7a53f02489fb9 100644 --- a/src/connector/src/source/nexmark/enumerator/mod.rs +++ b/src/connector/src/source/nexmark/enumerator/mod.rs @@ -14,6 +14,7 @@ use async_trait::async_trait; +use crate::error::ConnectorResult; use crate::source::nexmark::split::NexmarkSplit; use crate::source::nexmark::NexmarkProperties; use crate::source::{SourceEnumeratorContextRef, SplitEnumerator}; @@ -32,12 +33,12 @@ impl SplitEnumerator for NexmarkSplitEnumerator { async fn new( properties: NexmarkProperties, _context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> ConnectorResult { let split_num = properties.split_num; Ok(Self { split_num }) } - async fn list_splits(&mut self) -> anyhow::Result> { + async fn list_splits(&mut self) -> ConnectorResult> { let mut splits = vec![]; for i in 0..self.split_num { splits.push(NexmarkSplit { @@ -52,9 +53,8 @@ impl SplitEnumerator for NexmarkSplitEnumerator { #[cfg(test)] mod tests { - use anyhow::Result; - use super::*; + use crate::error::ConnectorResult as Result; use crate::source::SplitMetaData; #[tokio::test] diff --git a/src/connector/src/source/nexmark/source/combined_event.rs b/src/connector/src/source/nexmark/source/combined_event.rs index 7242c838f77f4..47a4d7e6f421d 100644 --- a/src/connector/src/source/nexmark/source/combined_event.rs +++ b/src/connector/src/source/nexmark/source/combined_event.rs @@ -16,7 +16,6 @@ pub use nexmark::event::EventType; use nexmark::event::{Auction, Bid, Event, Person}; use risingwave_common::array::StructValue; use risingwave_common::catalog::ROWID_PREFIX; -use risingwave_common::row::OwnedRow; use risingwave_common::types::{DataType, Datum, ScalarImpl, StructType, Timestamp}; use serde::{Deserialize, Serialize}; @@ -180,7 +179,10 @@ pub(crate) fn get_bid_struct_type() -> StructType { ]) } -pub(crate) fn combined_event_to_row(e: CombinedEvent, row_id_index: Option) -> OwnedRow { +pub(crate) fn combined_event_to_row( + e: CombinedEvent, + row_id_index: Option, +) -> Vec> { let mut fields = vec![ Some(ScalarImpl::Int64(e.event_type as i64)), e.person @@ -199,10 +201,10 @@ pub(crate) fn combined_event_to_row(e: CombinedEvent, row_id_index: Option) -> OwnedRow { +pub(crate) fn event_to_row(e: Event, row_id_index: Option) -> Vec> { let mut fields = match e { Event::Person(p) => person_to_datum(p), Event::Auction(a) => auction_to_datum(a), @@ -212,7 +214,7 @@ pub(crate) fn event_to_row(e: Event, row_id_index: Option) -> OwnedRow { // _row_id fields.insert(row_id_index, None); } - OwnedRow::new(fields) + fields } fn person_to_datum(p: Person) -> Vec { diff --git a/src/connector/src/source/nexmark/source/reader.rs b/src/connector/src/source/nexmark/source/reader.rs index 10c052c6d1b98..fd68348d6faf6 100644 --- a/src/connector/src/source/nexmark/source/reader.rs +++ b/src/connector/src/source/nexmark/source/reader.rs @@ -14,19 +14,20 @@ use std::time::Duration; -use anyhow::Result; use async_trait::async_trait; use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; -use maplit::hashmap; use nexmark::config::NexmarkConfig; use nexmark::event::EventType; use nexmark::EventGenerator; +use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::error::RwError; use risingwave_common::estimate_size::EstimateSize; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{DataType, ScalarImpl}; use tokio::time::Instant; +use crate::error::ConnectorResult; use crate::parser::ParserConfig; use crate::source::data_gen_util::spawn_data_generation_stream; use crate::source::nexmark::source::combined_event::{ @@ -34,8 +35,7 @@ use crate::source::nexmark::source::combined_event::{ }; use crate::source::nexmark::{NexmarkProperties, NexmarkSplit}; use crate::source::{ - BoxSourceWithStateStream, Column, SourceContextRef, SplitId, SplitMetaData, SplitReader, - StreamChunkWithState, + BoxChunkSourceStream, Column, SourceContextRef, SplitId, SplitMetaData, SplitReader, }; #[derive(Debug)] @@ -65,7 +65,7 @@ impl SplitReader for NexmarkSplitReader { parser_config: ParserConfig, source_ctx: SourceContextRef, _columns: Option>, - ) -> Result { + ) -> ConnectorResult { tracing::debug!("Splits for nexmark found! {:?}", splits); assert!(splits.len() == 1); // TODO: currently, assume there's only one split in one reader @@ -106,27 +106,40 @@ impl SplitReader for NexmarkSplitReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { let actor_id = self.source_ctx.source_info.actor_id.to_string(); + let fragment_id = self.source_ctx.source_info.fragment_id.to_string(); let source_id = self.source_ctx.source_info.source_id.to_string(); + let source_name = self.source_ctx.source_info.source_name.to_string(); let split_id = self.split_id.clone(); let metrics = self.source_ctx.metrics.clone(); // Will buffer at most 4 event chunks. const BUFFER_SIZE: usize = 4; spawn_data_generation_stream( - self.into_native_stream().inspect_ok( - move |chunk_with_states: &StreamChunkWithState| { + self.into_native_stream() + .inspect_ok(move |chunk: &StreamChunk| { metrics .partition_input_count - .with_label_values(&[&actor_id, &source_id, &split_id]) - .inc_by(chunk_with_states.chunk.cardinality() as u64); + .with_label_values(&[ + &actor_id, + &source_id, + &split_id, + &source_name, + &fragment_id, + ]) + .inc_by(chunk.cardinality() as u64); metrics .partition_input_bytes - .with_label_values(&[&actor_id, &source_id, &split_id]) - .inc_by(chunk_with_states.chunk.estimated_size() as u64); - }, - ), + .with_label_values(&[ + &actor_id, + &source_id, + &split_id, + &source_name, + &fragment_id, + ]) + .inc_by(chunk.estimated_size() as u64); + }), BUFFER_SIZE, ) .boxed() @@ -134,49 +147,60 @@ impl SplitReader for NexmarkSplitReader { } impl NexmarkSplitReader { - #[try_stream(boxed, ok = StreamChunkWithState, error = RwError)] + async fn sleep_til_next_event(&self, start_time: Instant, start_offset: u64, start_ts: u64) { + if self.use_real_time { + tokio::time::sleep_until( + start_time + Duration::from_millis(self.generator.timestamp() - start_ts), + ) + .await; + } else if self.min_event_gap_in_ns > 0 { + tokio::time::sleep_until( + start_time + + Duration::from_nanos( + self.min_event_gap_in_ns * (self.generator.global_offset() - start_offset), + ), + ) + .await; + } + } + + #[try_stream(boxed, ok = StreamChunk, error = crate::error::ConnectorError)] async fn into_native_stream(mut self) { let start_time = Instant::now(); let start_offset = self.generator.global_offset(); let start_ts = self.generator.timestamp(); - let event_dtypes = get_event_data_types(self.event_type, self.row_id_index); + let mut event_dtypes_with_offset = get_event_data_types(self.event_type, self.row_id_index); + event_dtypes_with_offset.extend([DataType::Varchar, DataType::Varchar]); + + let mut chunk_builder = + StreamChunkBuilder::new(self.max_chunk_size as usize, event_dtypes_with_offset); loop { - let mut rows = vec![]; - while (rows.len() as u64) < self.max_chunk_size { - if self.generator.global_offset() >= self.event_num { - break; - } - let event = self.generator.next().unwrap(); - let row = match self.event_type { - Some(_) => event_to_row(event, self.row_id_index), - None => combined_event_to_row(new_combined_event(event), self.row_id_index), - }; - rows.push((Op::Insert, row)); - } - if rows.is_empty() { + if self.generator.global_offset() >= self.event_num { break; } - if self.use_real_time { - tokio::time::sleep_until( - start_time + Duration::from_millis(self.generator.timestamp() - start_ts), - ) - .await; - } else if self.min_event_gap_in_ns > 0 { - tokio::time::sleep_until( - start_time - + Duration::from_nanos( - self.min_event_gap_in_ns - * (self.generator.global_offset() - start_offset), - ), - ) - .await; - } - let mapping = hashmap! {self.split_id.clone() => self.generator.offset().to_string()}; - let stream_chunk = StreamChunk::from_rows(&rows, &event_dtypes); - yield StreamChunkWithState { - chunk: stream_chunk, - split_offset_mapping: Some(mapping), + let event = self.generator.next().unwrap(); + let mut fields = match self.event_type { + Some(_) => event_to_row(event, self.row_id_index), + None => combined_event_to_row(new_combined_event(event), self.row_id_index), }; + fields.extend([ + Some(ScalarImpl::Utf8(self.split_id.as_ref().into())), + Some(ScalarImpl::Utf8( + self.generator.offset().to_string().into_boxed_str(), + )), + ]); + + if let Some(chunk) = chunk_builder.append_row(Op::Insert, OwnedRow::new(fields)) { + self.sleep_til_next_event(start_time, start_offset, start_ts) + .await; + yield chunk; + } + } + + if let Some(chunk) = chunk_builder.take() { + self.sleep_til_next_event(start_time, start_offset, start_ts) + .await; + yield chunk; } tracing::debug!(?self.event_type, "nexmark generator finished"); @@ -185,14 +209,12 @@ impl NexmarkSplitReader { #[cfg(test)] mod tests { - use anyhow::Result; - use super::*; use crate::source::nexmark::{NexmarkProperties, NexmarkSplitEnumerator}; use crate::source::{SourceEnumeratorContext, SplitEnumerator}; #[tokio::test] - async fn test_nexmark_split_reader() -> Result<()> { + async fn test_nexmark_split_reader() -> crate::error::ConnectorResult<()> { let props = NexmarkProperties { split_num: 2, min_event_gap_in_ns: 0, @@ -224,4 +246,45 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_nexmark_event_num() -> crate::error::ConnectorResult<()> { + let max_chunk_size = 32; + let event_num = max_chunk_size * 128 + 1; + let props = NexmarkProperties { + split_num: 1, + min_event_gap_in_ns: 0, + table_type: None, + max_chunk_size, + event_num, + ..Default::default() + }; + + let mut enumerator = + NexmarkSplitEnumerator::new(props.clone(), SourceEnumeratorContext::default().into()) + .await?; + let list_splits_resp: Vec<_> = enumerator.list_splits().await?.into_iter().collect(); + + for split in list_splits_resp { + let state = vec![split]; + let reader = NexmarkSplitReader::new( + props.clone(), + state, + Default::default(), + Default::default(), + None, + ) + .await? + .into_stream(); + let (rows_count, chunk_count) = reader + .fold((0, 0), |acc, x| async move { + (acc.0 + x.unwrap().cardinality(), acc.1 + 1) + }) + .await; + assert_eq!(rows_count as u64, event_num); + assert_eq!(chunk_count as u64, event_num / max_chunk_size + 1); + } + + Ok(()) + } } diff --git a/src/connector/src/source/nexmark/split.rs b/src/connector/src/source/nexmark/split.rs index d68aa2c8e1aa4..5150f1b6a1e1d 100644 --- a/src/connector/src/source/nexmark/split.rs +++ b/src/connector/src/source/nexmark/split.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::{SplitId, SplitMetaData}; #[derive(Clone, Debug, Default, Serialize, Deserialize, PartialEq, Hash)] @@ -31,15 +31,15 @@ impl SplitMetaData for NexmarkSplit { format!("{}-{}", self.split_num, self.split_index).into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { self.start_offset = Some(start_offset.as_str().parse::().unwrap()); Ok(()) } diff --git a/src/connector/src/source/pulsar/enumerator/client.rs b/src/connector/src/source/pulsar/enumerator/client.rs index d92e633060616..dddcf927d0c3f 100644 --- a/src/connector/src/source/pulsar/enumerator/client.rs +++ b/src/connector/src/source/pulsar/enumerator/client.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{anyhow, bail, Result}; +use anyhow::anyhow; use async_trait::async_trait; use itertools::Itertools; use pulsar::{Pulsar, TokioExecutor}; +use risingwave_common::bail; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::pulsar::split::PulsarSplit; use crate::source::pulsar::topic::{parse_topic, Topic}; use crate::source::pulsar::PulsarProperties; @@ -45,7 +47,7 @@ impl SplitEnumerator for PulsarSplitEnumerator { async fn new( properties: PulsarProperties, _context: SourceEnumeratorContextRef, - ) -> Result { + ) -> ConnectorResult { let pulsar = properties .common .build_client(&properties.oauth, &properties.aws_auth_props) @@ -80,7 +82,7 @@ impl SplitEnumerator for PulsarSplitEnumerator { }) } - async fn list_splits(&mut self) -> anyhow::Result> { + async fn list_splits(&mut self) -> ConnectorResult> { let offset = self.start_offset.clone(); // MessageId is only used when recovering from a State assert!(!matches!(offset, PulsarEnumeratorOffset::MessageId(_))); diff --git a/src/connector/src/source/pulsar/source/reader.rs b/src/connector/src/source/pulsar/source/reader.rs index 8d80487a7da8b..9ed810dfc933a 100644 --- a/src/connector/src/source/pulsar/source/reader.rs +++ b/src/connector/src/source/pulsar/source/reader.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::time::{SystemTime, UNIX_EPOCH}; -use anyhow::{anyhow, ensure, Result}; +use anyhow::Context; use arrow_array::{Int32Array, Int64Array, RecordBatch}; use async_trait::async_trait; use futures::StreamExt; @@ -30,15 +30,16 @@ use pulsar::message::proto::MessageIdData; use pulsar::{Consumer, ConsumerBuilder, ConsumerOptions, Pulsar, SubType, TokioExecutor}; use risingwave_common::array::{DataChunk, StreamChunk}; use risingwave_common::catalog::ROWID_PREFIX; -use risingwave_common::error::RwError; +use risingwave_common::{bail, ensure}; +use thiserror_ext::AsReport; -use crate::error::ConnectorError; +use crate::error::ConnectorResult; use crate::parser::ParserConfig; use crate::source::pulsar::split::PulsarSplit; use crate::source::pulsar::{PulsarEnumeratorOffset, PulsarProperties}; use crate::source::{ - into_chunk_stream, BoxSourceWithStateStream, Column, CommonSplitReader, SourceContextRef, - SourceMessage, SplitId, SplitMetaData, SplitReader, StreamChunkWithState, + into_chunk_stream, BoxChunkSourceStream, Column, CommonSplitReader, SourceContextRef, + SourceMessage, SplitId, SplitMetaData, SplitReader, }; pub enum PulsarSplitReader { @@ -57,7 +58,7 @@ impl SplitReader for PulsarSplitReader { parser_config: ParserConfig, source_ctx: SourceContextRef, _columns: Option>, - ) -> Result { + ) -> ConnectorResult { ensure!(splits.len() == 1, "only support single split"); let split = splits.into_iter().next().unwrap(); let topic = split.topic.to_string(); @@ -83,7 +84,7 @@ impl SplitReader for PulsarSplitReader { } } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { match self { Self::Broker(reader) => { let (parser_config, source_context) = @@ -107,19 +108,15 @@ pub struct PulsarBrokerReader { } // {ledger_id}:{entry_id}:{partition}:{batch_index} -fn parse_message_id(id: &str) -> Result { +fn parse_message_id(id: &str) -> ConnectorResult { let splits = id.split(':').collect_vec(); if splits.len() < 2 || splits.len() > 4 { - return Err(anyhow!("illegal message id string {}", id)); + bail!("illegal message id string {}", id); } - let ledger_id = splits[0] - .parse::() - .map_err(|e| anyhow!("illegal ledger id {}", e))?; - let entry_id = splits[1] - .parse::() - .map_err(|e| anyhow!("illegal entry id {}", e))?; + let ledger_id = splits[0].parse::().context("illegal ledger id")?; + let entry_id = splits[1].parse::().context("illegal entry id")?; let mut message_id = MessageIdData { ledger_id, @@ -132,16 +129,12 @@ fn parse_message_id(id: &str) -> Result { }; if splits.len() > 2 { - let partition = splits[2] - .parse::() - .map_err(|e| anyhow!("illegal partition {}", e))?; + let partition = splits[2].parse::().context("illegal partition")?; message_id.partition = Some(partition); } if splits.len() == 4 { - let batch_index = splits[3] - .parse::() - .map_err(|e| anyhow!("illegal batch index {}", e))?; + let batch_index = splits[3].parse::().context("illegal batch index")?; message_id.batch_index = Some(batch_index); } @@ -159,7 +152,7 @@ impl SplitReader for PulsarBrokerReader { parser_config: ParserConfig, source_ctx: SourceContextRef, _columns: Option>, - ) -> Result { + ) -> ConnectorResult { ensure!(splits.len() == 1, "only support single split"); let split = splits.into_iter().next().unwrap(); let pulsar = props @@ -234,7 +227,7 @@ impl SplitReader for PulsarBrokerReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { let parser_config = self.parser_config.clone(); let source_context = self.source_ctx.clone(); into_chunk_stream(self, parser_config, source_context) @@ -242,7 +235,7 @@ impl SplitReader for PulsarBrokerReader { } impl CommonSplitReader for PulsarBrokerReader { - #[try_stream(ok = Vec, error = anyhow::Error)] + #[try_stream(ok = Vec, error = crate::error::ConnectorError)] async fn into_data_stream(self) { let max_chunk_size = self.source_ctx.source_ctrl_opts.chunk_size; #[for_await] @@ -287,7 +280,7 @@ impl PulsarIcebergReader { } } - async fn scan(&self) -> Result { + async fn scan(&self) -> ConnectorResult { let table = self.create_iceberg_table().await?; let schema = table.current_table_metadata().current_schema()?; tracing::debug!("Created iceberg pulsar table, schema is: {:?}", schema,); @@ -302,18 +295,14 @@ impl PulsarIcebergReader { .fields() .iter() .find(|f| f.name == META_COLUMN_PARTITION) - .ok_or_else(|| { - ConnectorError::Pulsar(anyhow!( - "Partition field not found in partition spec" - )) - })?; + .context("Partition field not found in partition spec")?; (s.clone(), field.clone()) } _ => { - return Err(ConnectorError::Pulsar(anyhow!( + bail!( "Partition type is not struct in iceberg table: {}", table.table_name() - )))?; + ); } }; @@ -334,29 +323,27 @@ impl PulsarIcebergReader { .new_scan_builder() .with_partition_value(partition_value) .with_batch_size(max_chunk_size) - .build()? + .build() + .context("failed to build iceberg table scan")? .scan(&table) .await?) } - async fn create_iceberg_table(&self) -> Result

{ + async fn create_iceberg_table(&self) -> ConnectorResult
{ let catalog = load_catalog(&self.build_iceberg_configs()?) .await - .map_err(|e| ConnectorError::Pulsar(anyhow!("Unable to load iceberg catalog: {e}")))?; + .context("Unable to load iceberg catalog")?; let table_id = TableIdentifier::new(vec![self.split.topic.topic_str_without_partition()?]) - .map_err(|e| ConnectorError::Pulsar(anyhow!("Unable to parse table name: {e}")))?; + .context("Unable to parse table name")?; - let table = catalog - .load_table(&table_id) - .await - .map_err(|err| ConnectorError::Pulsar(anyhow!(err)))?; + let table = catalog.load_table(&table_id).await?; Ok(table) } - #[try_stream(ok = StreamChunkWithState, error = anyhow::Error)] + #[try_stream(ok = (StreamChunk, HashMap), error = crate::error::ConnectorError)] async fn as_stream_chunk_stream(&self) { #[for_await] for file_scan in self.scan().await? { @@ -371,7 +358,7 @@ impl PulsarIcebergReader { } } - #[try_stream(ok = StreamChunkWithState, error = RwError)] + #[try_stream(ok = StreamChunk, error = crate::error::ConnectorError)] async fn into_stream(self) { let (props, mut split, parser_config, source_ctx) = ( self.props.clone(), @@ -384,13 +371,10 @@ impl PulsarIcebergReader { #[for_await] for msg in self.as_stream_chunk_stream() { - let msg = - msg.inspect_err(|e| tracing::error!("Failed to read message from iceberg: {}", e))?; - last_msg_id = msg - .split_offset_mapping - .as_ref() - .and_then(|m| m.get(self.split.topic.to_string().as_str())) - .cloned(); + let (_chunk, mapping) = msg.inspect_err( + |e| tracing::error!(error = %e.as_report(), "Failed to read message from iceberg"), + )?; + last_msg_id = mapping.get(self.split.topic.to_string().as_str()).cloned(); } tracing::info!("Finished reading pulsar message from iceberg"); @@ -414,13 +398,14 @@ impl PulsarIcebergReader { } } - fn build_iceberg_configs(&self) -> Result> { + fn build_iceberg_configs(&self) -> ConnectorResult> { let mut iceberg_configs = HashMap::new(); - let bucket = - self.props.iceberg_bucket.as_ref().ok_or_else(|| { - ConnectorError::Pulsar(anyhow!("Iceberg bucket is not configured")) - })?; + let bucket = self + .props + .iceberg_bucket + .as_ref() + .context("Iceberg bucket is not configured")?; iceberg_configs.insert(CATALOG_TYPE.to_string(), "storage".to_string()); iceberg_configs.insert(CATALOG_NAME.to_string(), "pulsar".to_string()); @@ -470,58 +455,50 @@ impl PulsarIcebergReader { fn convert_record_batch_to_source_with_state( &self, record_batch: &RecordBatch, - ) -> Result { + ) -> ConnectorResult<(StreamChunk, HashMap)> { let mut offsets = Vec::with_capacity(record_batch.num_rows()); let ledger_id_array = record_batch .column_by_name(META_COLUMN_LEDGER_ID) - .ok_or_else(|| ConnectorError::Pulsar(anyhow!("Ledger id not found in iceberg table")))? + .context("Ledger id not found in iceberg table")? .as_any() .downcast_ref::() - .ok_or_else(|| { - ConnectorError::Pulsar(anyhow!("Ledger id is not i64 in iceberg table")) - })?; + .context("Ledger id is not i64 in iceberg table")?; let entry_id_array = record_batch .column_by_name(META_COLUMN_ENTRY_ID) - .ok_or_else(|| ConnectorError::Pulsar(anyhow!("Entry id not found in iceberg table")))? + .context("Entry id not found in iceberg table")? .as_any() .downcast_ref::() - .ok_or_else(|| { - ConnectorError::Pulsar(anyhow!("Entry id is not i64 in iceberg table")) - })?; + .context("Entry id is not i64 in iceberg table")?; let partition_array = record_batch .column_by_name(META_COLUMN_PARTITION) .map(|arr| { - arr.as_any().downcast_ref::().ok_or_else(|| { - ConnectorError::Pulsar(anyhow!("Partition is not i32 in iceberg table")) - }) + arr.as_any() + .downcast_ref::() + .context("Partition is not i32 in iceberg table") }) .transpose()?; let batch_index_array = record_batch .column_by_name(META_COLUMN_BATCH_INDEX) .map(|arr| { - arr.as_any().downcast_ref::().ok_or_else(|| { - ConnectorError::Pulsar(anyhow!("Batch index is not i64 in iceberg table")) - }) + arr.as_any() + .downcast_ref::() + .context("Batch index is not i64 in iceberg table") }) .transpose()?; - let field_indices = self + let field_indices: Vec<_> = self .parser_config .common .rw_columns .iter() .filter(|col| col.name != ROWID_PREFIX) - .map(|col| { - record_batch - .schema() - .index_of(col.name.as_str()) - .map_err(|e| anyhow!(e)) - }) - .collect::>>()?; + .map(|col| record_batch.schema().index_of(col.name.as_str())) + .try_collect() + .context("failed to look up column name in arrow record batch")?; for row in 0..record_batch.num_rows() { let offset = format!( @@ -535,18 +512,16 @@ impl PulsarIcebergReader { offsets.push(offset); } - let data_chunk = DataChunk::try_from(&record_batch.project(&field_indices)?)?; + let data_chunk = DataChunk::try_from(&record_batch.project(&field_indices)?) + .context("failed to convert arrow record batch to data chunk")?; let stream_chunk = StreamChunk::from(data_chunk); - let state = Some(HashMap::from([( + let state = HashMap::from([( self.split.topic.to_string().into(), offsets.last().unwrap().clone(), - )])); + )]); - Ok(StreamChunkWithState { - chunk: stream_chunk, - split_offset_mapping: state, - }) + Ok((stream_chunk, state)) } } diff --git a/src/connector/src/source/pulsar/split.rs b/src/connector/src/source/pulsar/split.rs index 36f9bc47e3ec5..bf9b63d99d74f 100644 --- a/src/connector/src/source/pulsar/split.rs +++ b/src/connector/src/source/pulsar/split.rs @@ -12,10 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::anyhow; use risingwave_common::types::JsonbVal; use serde::{Deserialize, Serialize}; +use crate::error::ConnectorResult; use crate::source::pulsar::topic::Topic; use crate::source::pulsar::PulsarEnumeratorOffset; use crate::source::{SplitId, SplitMetaData}; @@ -32,15 +32,15 @@ impl SplitMetaData for PulsarSplit { self.topic.to_string().into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } fn encode_to_json(&self) -> JsonbVal { serde_json::to_value(self.clone()).unwrap().into() } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { let start_offset = if start_offset.is_empty() { PulsarEnumeratorOffset::Earliest } else { diff --git a/src/connector/src/source/pulsar/topic.rs b/src/connector/src/source/pulsar/topic.rs index 8c6ec41022b0a..352c7e47d8da2 100644 --- a/src/connector/src/source/pulsar/topic.rs +++ b/src/connector/src/source/pulsar/topic.rs @@ -12,10 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use anyhow::{anyhow, Result}; +use anyhow::anyhow; +use risingwave_common::bail; use serde::{Deserialize, Serialize}; use urlencoding::encode; +use crate::error::ConnectorResult as Result; + const PERSISTENT_DOMAIN: &str = "persistent"; const NON_PERSISTENT_DOMAIN: &str = "non-persistent"; const PUBLIC_TENANT: &str = "public"; @@ -32,9 +35,10 @@ pub struct Topic { pub partition_index: Option, } -impl ToString for Topic { - fn to_string(&self) -> String { - format!( +impl std::fmt::Display for Topic { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, "{}://{}/{}/{}", self.domain, self.tenant, self.namespace, self.topic ) @@ -58,7 +62,7 @@ impl Topic { pub fn sub_topic(&self, partition: i32) -> Result { if partition < 0 { - return Err(anyhow!("invalid partition index number")); + bail!("invalid partition index number"); } if self.topic.contains(PARTITIONED_TOPIC_SUFFIX) { @@ -118,11 +122,11 @@ pub fn parse_topic(topic: &str) -> Result { ), 3 => format!("{}://{}", PERSISTENT_DOMAIN, topic), _ => { - return Err(anyhow!( + bail!( "Invalid short topic name '{}', \ it should be in the format of // or ", topic - )); + ); } }; } @@ -132,10 +136,10 @@ pub fn parse_topic(topic: &str) -> Result { let domain = match parts[0] { PERSISTENT_DOMAIN | NON_PERSISTENT_DOMAIN => parts[0], _ => { - return Err(anyhow!( + bail!( "The domain only can be specified as 'persistent' or 'non-persistent'. Input domain is '{}'", parts[0] - )); + ); } }; @@ -143,10 +147,10 @@ pub fn parse_topic(topic: &str) -> Result { let parts: Vec<&str> = rest.splitn(3, '/').collect(); if parts.len() != 3 { - return Err(anyhow!( + bail!( "invalid topic name '{}', it should be in the format of //", rest - )); + ); } let parsed_topic = Topic { @@ -158,7 +162,7 @@ pub fn parse_topic(topic: &str) -> Result { }; if parsed_topic.topic.is_empty() { - return Err(anyhow!("topic name cannot be empty".to_string(),)); + bail!("topic name cannot be empty".to_string()); } Ok(parsed_topic) diff --git a/src/source/src/source_desc.rs b/src/connector/src/source/reader/desc.rs similarity index 56% rename from src/source/src/source_desc.rs rename to src/connector/src/source/reader/desc.rs index aed602aa71ff8..46107c2d73d0a 100644 --- a/src/source/src/source_desc.rs +++ b/src/connector/src/source/reader/desc.rs @@ -15,33 +15,42 @@ use std::collections::HashMap; use std::sync::Arc; -use risingwave_common::catalog::ColumnDesc; -use risingwave_common::error::ErrorCode::ProtocolError; -use risingwave_common::error::{Result, RwError}; -use risingwave_connector::parser::{EncodingProperties, ProtocolProperties, SpecificParserConfig}; -use risingwave_connector::source::monitor::SourceMetrics; -use risingwave_connector::source::{SourceColumnDesc, SourceColumnType}; -use risingwave_connector::ConnectorParams; +use risingwave_common::bail; +use risingwave_common::catalog::{ColumnDesc, ColumnId}; +use risingwave_common::util::iter_util::ZipEqFast; use risingwave_pb::catalog::PbStreamSourceInfo; -use risingwave_pb::plan_common::PbColumnCatalog; +use risingwave_pb::plan_common::additional_column::ColumnType; +use risingwave_pb::plan_common::{AdditionalColumn, PbColumnCatalog}; -use crate::connector_source::ConnectorSource; -use crate::fs_connector_source::FsConnectorSource; +#[expect(deprecated)] +use super::fs_reader::FsSourceReader; +use super::reader::SourceReader; +use crate::error::ConnectorResult; +use crate::parser::additional_columns::{ + build_additional_column_catalog, COMMON_COMPATIBLE_ADDITIONAL_COLUMNS, + COMPATIBLE_ADDITIONAL_COLUMNS, +}; +use crate::parser::{EncodingProperties, ProtocolProperties, SpecificParserConfig}; +use crate::source::monitor::SourceMetrics; +use crate::source::{SourceColumnDesc, SourceColumnType, UPSTREAM_SOURCE_KEY}; +use crate::ConnectorParams; pub const DEFAULT_CONNECTOR_MESSAGE_BUFFER_SIZE: usize = 16; /// `SourceDesc` describes a stream source. #[derive(Debug, Clone)] pub struct SourceDesc { - pub source: ConnectorSource, + pub source: SourceReader, pub columns: Vec, pub metrics: Arc, } /// `FsSourceDesc` describes a stream source. +#[deprecated = "will be replaced by new fs source (list + fetch)"] +#[expect(deprecated)] #[derive(Debug)] pub struct FsSourceDesc { - pub source: FsConnectorSource, + pub source: FsSourceReader, pub columns: Vec, pub metrics: Arc, } @@ -82,12 +91,83 @@ impl SourceDescBuilder { } } - fn column_catalogs_to_source_column_descs(&self) -> Vec { + /// This function builds `SourceColumnDesc` from `ColumnCatalog`, and handle the creation + /// of hidden columns like partition/file, offset that are not specified by user. + pub fn column_catalogs_to_source_column_descs(&self) -> Vec { + let mut columns_exist = [false; 2]; + let mut last_column_id = self + .columns + .iter() + .map(|c| c.column_desc.as_ref().unwrap().column_id.into()) + .max() + .unwrap_or(ColumnId::placeholder()); + let connector_name = self + .with_properties + .get(UPSTREAM_SOURCE_KEY) + .map(|s| s.to_lowercase()) + .unwrap(); + + let additional_columns: Vec<_> = { + let compat_col_types = COMPATIBLE_ADDITIONAL_COLUMNS + .get(&*connector_name) + .unwrap_or(&COMMON_COMPATIBLE_ADDITIONAL_COLUMNS); + ["partition", "file", "offset"] + .iter() + .filter_map(|col_type| { + last_column_id = last_column_id.next(); + if compat_col_types.contains(col_type) { + Some( + build_additional_column_catalog( + last_column_id, + &connector_name, + col_type, + None, + None, + None, + false, + ) + .unwrap() + .to_protobuf(), + ) + } else { + None + } + }) + .collect() + }; + assert_eq!(additional_columns.len(), 2); + + // Check if partition/file/offset columns are included explicitly. + for col in &self.columns { + match col.column_desc.as_ref().unwrap().get_additional_column() { + Ok(AdditionalColumn { + column_type: Some(ColumnType::Partition(_) | ColumnType::Filename(_)), + }) => { + columns_exist[0] = true; + } + Ok(AdditionalColumn { + column_type: Some(ColumnType::Offset(_)), + }) => { + columns_exist[1] = true; + } + _ => (), + } + } + let mut columns: Vec<_> = self .columns .iter() .map(|c| SourceColumnDesc::from(&ColumnDesc::from(c.column_desc.as_ref().unwrap()))) .collect(); + + for (existed, c) in columns_exist.iter().zip_eq_fast(&additional_columns) { + if !existed { + columns.push(SourceColumnDesc::hidden_addition_col_from_column_desc( + &ColumnDesc::from(c.column_desc.as_ref().unwrap()), + )); + } + } + if let Some(row_id_index) = self.row_id_index { columns[row_id_index].column_type = SourceColumnType::RowId; } @@ -97,12 +177,12 @@ impl SourceDescBuilder { columns } - pub fn build(self) -> Result { + pub fn build(self) -> ConnectorResult { let columns = self.column_catalogs_to_source_column_descs(); let psrser_config = SpecificParserConfig::new(&self.source_info, &self.with_properties)?; - let source = ConnectorSource::new( + let source = SourceReader::new( self.with_properties, columns.clone(), self.connector_message_buffer_size, @@ -120,7 +200,9 @@ impl SourceDescBuilder { self.metrics.clone() } - pub fn build_fs_source_desc(&self) -> Result { + #[deprecated = "will be replaced by new fs source (list + fetch)"] + #[expect(deprecated)] + pub fn build_fs_source_desc(&self) -> ConnectorResult { let parser_config = SpecificParserConfig::new(&self.source_info, &self.with_properties)?; match ( @@ -132,16 +214,17 @@ impl SourceDescBuilder { EncodingProperties::Csv(_) | EncodingProperties::Json(_), ) => {} (format, encode) => { - return Err(RwError::from(ProtocolError(format!( + bail!( "Unsupported combination of format {:?} and encode {:?}", - format, encode - )))); + format, + encode, + ); } } let columns = self.column_catalogs_to_source_column_descs(); - let source = FsConnectorSource::new( + let source = FsSourceReader::new( self.with_properties.clone(), columns.clone(), self.connector_params diff --git a/src/source/src/fs_connector_source.rs b/src/connector/src/source/reader/fs_reader.rs similarity index 73% rename from src/source/src/fs_connector_source.rs rename to src/connector/src/source/reader/fs_reader.rs index a44fc0a9792a2..93a0bd2c2d6a8 100644 --- a/src/source/src/fs_connector_source.rs +++ b/src/connector/src/source/reader/fs_reader.rs @@ -12,46 +12,45 @@ // See the License for the specific language governing permissions and // limitations under the License. -// *** NOTICE: TO BE DEPRECATED *** // +#![deprecated = "will be replaced by new fs source (list + fetch)"] use std::collections::HashMap; use std::sync::Arc; -use anyhow::anyhow; +use anyhow::Context; use futures::stream::pending; use futures::StreamExt; use risingwave_common::catalog::ColumnId; -use risingwave_common::error::ErrorCode::ConnectorError; -use risingwave_common::error::Result; -use risingwave_connector::dispatch_source_prop; -use risingwave_connector::parser::{CommonParserConfig, ParserConfig, SpecificParserConfig}; -use risingwave_connector::source::{ - create_split_reader, BoxSourceWithStateStream, ConnectorProperties, ConnectorState, + +use crate::dispatch_source_prop; +use crate::error::ConnectorResult; +use crate::parser::{CommonParserConfig, ParserConfig, SpecificParserConfig}; +use crate::source::{ + create_split_reader, BoxChunkSourceStream, ConnectorProperties, ConnectorState, SourceColumnDesc, SourceContext, SplitReader, }; #[derive(Clone, Debug)] -pub struct FsConnectorSource { +pub struct FsSourceReader { pub config: ConnectorProperties, pub columns: Vec, pub properties: HashMap, pub parser_config: SpecificParserConfig, } -impl FsConnectorSource { +impl FsSourceReader { #[allow(clippy::too_many_arguments)] pub fn new( properties: HashMap, columns: Vec, connector_node_addr: Option, parser_config: SpecificParserConfig, - ) -> Result { + ) -> ConnectorResult { // Store the connector node address to properties for later use. let mut source_props: HashMap = HashMap::from_iter(properties.clone()); connector_node_addr .map(|addr| source_props.insert("connector_node_addr".to_string(), addr)); - let config = ConnectorProperties::extract(source_props, false) - .map_err(|e| ConnectorError(e.into()))?; + let config = ConnectorProperties::extract(source_props, false)?; Ok(Self { config, @@ -61,27 +60,31 @@ impl FsConnectorSource { }) } - fn get_target_columns(&self, column_ids: Vec) -> Result> { + fn get_target_columns( + &self, + column_ids: Vec, + ) -> ConnectorResult> { column_ids .iter() .map(|id| { self.columns .iter() .find(|c| c.column_id == *id) - .ok_or_else(|| { - anyhow!("Failed to find column id: {} in source: {:?}", id, self).into() + .with_context(|| { + format!("Failed to find column id: {} in source: {:?}", id, self) }) - .map(|col| col.clone()) + .cloned() }) - .collect::>>() + .try_collect() + .map_err(Into::into) } - pub async fn stream_reader( + pub async fn to_stream( &self, state: ConnectorState, column_ids: Vec, source_ctx: Arc, - ) -> Result { + ) -> ConnectorResult { let config = self.config.clone(); let columns = self.get_target_columns(column_ids)?; diff --git a/src/stream/src/executor/managed_state/mod.rs b/src/connector/src/source/reader/mod.rs similarity index 87% rename from src/stream/src/executor/managed_state/mod.rs rename to src/connector/src/source/reader/mod.rs index c32dfb11be7c6..fc6800a0f1da7 100644 --- a/src/stream/src/executor/managed_state/mod.rs +++ b/src/connector/src/source/reader/mod.rs @@ -12,4 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub mod join; +pub mod desc; +pub mod fs_reader; +#[expect(clippy::module_inception)] +pub mod reader; diff --git a/src/source/src/connector_source.rs b/src/connector/src/source/reader/reader.rs similarity index 71% rename from src/source/src/connector_source.rs rename to src/connector/src/source/reader/reader.rs index 441a91836bb0a..5cfd10835998a 100644 --- a/src/source/src/connector_source.rs +++ b/src/connector/src/source/reader/reader.rs @@ -15,7 +15,7 @@ use std::collections::HashMap; use std::sync::Arc; -use anyhow::anyhow; +use anyhow::Context; use futures::future::try_join_all; use futures::stream::pending; use futures::StreamExt; @@ -23,49 +23,38 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::bail; use risingwave_common::catalog::ColumnId; -use risingwave_common::error::ErrorCode::ConnectorError; -use risingwave_common::error::{Result, RwError}; -use risingwave_common::util::select_all; -use risingwave_connector::dispatch_source_prop; -use risingwave_connector::parser::{CommonParserConfig, ParserConfig, SpecificParserConfig}; -use risingwave_connector::source::filesystem::opendal_source::opendal_enumerator::OpendalEnumerator; -use risingwave_connector::source::filesystem::opendal_source::{ +use rw_futures_util::select_all; +use thiserror_ext::AsReport as _; + +use crate::dispatch_source_prop; +use crate::error::ConnectorResult; +use crate::parser::{CommonParserConfig, ParserConfig, SpecificParserConfig}; +use crate::source::filesystem::opendal_source::opendal_enumerator::OpendalEnumerator; +use crate::source::filesystem::opendal_source::{ OpendalGcs, OpendalPosixFs, OpendalS3, OpendalSource, }; -use risingwave_connector::source::filesystem::FsPageItem; -use risingwave_connector::source::{ - create_split_reader, BoxSourceWithStateStream, BoxTryStream, Column, ConnectorProperties, - ConnectorState, FsFilterCtrlCtx, SourceColumnDesc, SourceContext, SplitReader, +use crate::source::filesystem::{FsPageItem, OpendalFsSplit}; +use crate::source::{ + create_split_reader, BoxChunkSourceStream, BoxTryStream, Column, ConnectorProperties, + ConnectorState, SourceColumnDesc, SourceContext, SplitReader, }; -use tokio::time; -use tokio::time::Duration; #[derive(Clone, Debug)] -pub struct ConnectorSource { +pub struct SourceReader { pub config: ConnectorProperties, pub columns: Vec, pub parser_config: SpecificParserConfig, pub connector_message_buffer_size: usize, } -#[derive(Clone, Debug)] -pub struct FsListCtrlContext { - pub interval: Duration, - pub last_tick: Option, - - pub filter_ctx: FsFilterCtrlCtx, -} -pub type FsListCtrlContextRef = Arc; - -impl ConnectorSource { +impl SourceReader { pub fn new( properties: HashMap, columns: Vec, connector_message_buffer_size: usize, parser_config: SpecificParserConfig, - ) -> Result { - let config = ConnectorProperties::extract(properties, false) - .map_err(|e| ConnectorError(e.into()))?; + ) -> ConnectorResult { + let config = ConnectorProperties::extract(properties, false)?; Ok(Self { config, @@ -75,22 +64,26 @@ impl ConnectorSource { }) } - fn get_target_columns(&self, column_ids: Vec) -> Result> { + fn get_target_columns( + &self, + column_ids: Vec, + ) -> ConnectorResult> { column_ids .iter() .map(|id| { self.columns .iter() .find(|c| c.column_id == *id) - .ok_or_else(|| { - anyhow!("Failed to find column id: {} in source: {:?}", id, self).into() + .with_context(|| { + format!("Failed to find column id: {} in source: {:?}", id, self) }) - .map(|col| col.clone()) + .cloned() }) - .collect::>>() + .try_collect() + .map_err(Into::into) } - pub fn get_source_list(&self) -> Result> { + pub fn get_source_list(&self) -> ConnectorResult> { let config = self.config.clone(); match config { ConnectorProperties::Gcs(prop) => { @@ -112,12 +105,12 @@ impl ConnectorSource { } } - pub async fn stream_reader( + pub async fn to_stream( &self, state: ConnectorState, column_ids: Vec, source_ctx: Arc, - ) -> Result { + ) -> ConnectorResult { let Some(splits) = state else { return Ok(pending().boxed()); }; @@ -156,7 +149,6 @@ impl ConnectorSource { vec![reader] } else { let to_reader_splits = splits.into_iter().map(|split| vec![split]); - try_join_all(to_reader_splits.into_iter().map(|splits| { tracing::debug!(?splits, ?prop, "spawning connector split reader"); let props = prop.clone(); @@ -175,7 +167,7 @@ impl ConnectorSource { } } -#[try_stream(boxed, ok = FsPageItem, error = RwError)] +#[try_stream(boxed, ok = FsPageItem, error = crate::error::ConnectorError)] async fn build_opendal_fs_list_stream(lister: OpendalEnumerator) { let matcher = lister.get_matcher(); let mut object_metadata_iter = lister.list().await?; @@ -195,8 +187,35 @@ async fn build_opendal_fs_list_stream(lister: OpendalEnumera } } Err(err) => { - tracing::error!("list object fail, err {}", err); - return Err(err.into()); + tracing::error!(error = %err.as_report(), "list object fail"); + return Err(err); + } + } + } +} + +#[try_stream(boxed, ok = OpendalFsSplit, error = crate::error::ConnectorError)] +pub async fn build_opendal_fs_list_for_batch(lister: OpendalEnumerator) { + let matcher = lister.get_matcher(); + let mut object_metadata_iter = lister.list().await?; + + while let Some(list_res) = object_metadata_iter.next().await { + match list_res { + Ok(res) => { + if matcher + .as_ref() + .map(|m| m.matches(&res.name)) + .unwrap_or(true) + { + let split = OpendalFsSplit::new(res.name, 0, res.size as usize); + yield split + } else { + continue; + } + } + Err(err) => { + tracing::error!(error = %err.as_report(), "list object fail"); + return Err(err); } } } diff --git a/src/connector/src/source/test_source.rs b/src/connector/src/source/test_source.rs index 6c10ff9934eef..6d224593d7a2e 100644 --- a/src/connector/src/source/test_source.rs +++ b/src/connector/src/source/test_source.rs @@ -15,24 +15,25 @@ use std::collections::HashMap; use std::sync::{Arc, OnceLock}; -use anyhow::anyhow; use async_trait::async_trait; use parking_lot::Mutex; +use risingwave_common::bail; use risingwave_common::types::JsonbVal; use serde_derive::{Deserialize, Serialize}; use with_options::WithOptions; +use crate::error::ConnectorResult; use crate::parser::ParserConfig; use crate::source::{ - BoxSourceWithStateStream, Column, SourceContextRef, SourceEnumeratorContextRef, - SourceProperties, SplitEnumerator, SplitId, SplitMetaData, SplitReader, TryFromHashmap, + BoxChunkSourceStream, Column, SourceContextRef, SourceEnumeratorContextRef, SourceProperties, + SplitEnumerator, SplitId, SplitMetaData, SplitReader, TryFromHashmap, }; pub type BoxListSplits = Box< dyn FnMut( TestSourceProperties, SourceEnumeratorContextRef, - ) -> anyhow::Result> + ) -> ConnectorResult> + Send + 'static, >; @@ -44,7 +45,7 @@ pub type BoxIntoSourceStream = Box< ParserConfig, SourceContextRef, Option>, - ) -> BoxSourceWithStateStream + ) -> BoxChunkSourceStream + Send + 'static, >; @@ -59,7 +60,7 @@ impl BoxSource { list_splits: impl FnMut( TestSourceProperties, SourceEnumeratorContextRef, - ) -> anyhow::Result> + ) -> ConnectorResult> + Send + 'static, into_source_stream: impl FnMut( @@ -68,7 +69,7 @@ impl BoxSource { ParserConfig, SourceContextRef, Option>, - ) -> BoxSourceWithStateStream + ) -> BoxChunkSourceStream + Send + 'static, ) -> BoxSource { @@ -124,11 +125,11 @@ impl TryFromHashmap for TestSourceProperties { fn try_from_hashmap( props: HashMap, _deny_unknown_fields: bool, - ) -> anyhow::Result { + ) -> ConnectorResult { if cfg!(any(madsim, test)) { Ok(TestSourceProperties { properties: props }) } else { - Err(anyhow!("test source only available at test")) + bail!("test source only available at test") } } } @@ -149,11 +150,11 @@ impl SplitMetaData for TestSourceSplit { serde_json::to_value(self.clone()).unwrap().into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } - fn update_with_offset(&mut self, start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, start_offset: String) -> ConnectorResult<()> { self.offset = start_offset; Ok(()) } @@ -172,14 +173,14 @@ impl SplitEnumerator for TestSourceSplitEnumerator { async fn new( properties: Self::Properties, context: SourceEnumeratorContextRef, - ) -> anyhow::Result { + ) -> ConnectorResult { Ok(Self { properties, context, }) } - async fn list_splits(&mut self) -> anyhow::Result> { + async fn list_splits(&mut self) -> ConnectorResult> { (get_registry() .box_source .lock() @@ -208,7 +209,7 @@ impl SplitReader for TestSourceSplitReader { parser_config: ParserConfig, source_ctx: SourceContextRef, columns: Option>, - ) -> anyhow::Result { + ) -> ConnectorResult { Ok(Self { properties, state, @@ -218,7 +219,7 @@ impl SplitReader for TestSourceSplitReader { }) } - fn into_stream(self) -> BoxSourceWithStateStream { + fn into_stream(self) -> BoxChunkSourceStream { (get_registry() .box_source .lock() diff --git a/src/connector/with_options_sink.yaml b/src/connector/with_options_sink.yaml index d6a4d054b9b9f..35b06ec33fb76 100644 --- a/src/connector/with_options_sink.yaml +++ b/src/connector/with_options_sink.yaml @@ -123,7 +123,10 @@ IcebergConfig: required: true - name: database.name field_type: String - required: true + required: false + - name: catalog.name + field_type: String + required: false - name: catalog.type field_type: String required: false @@ -149,15 +152,15 @@ IcebergConfig: field_type: Vec required: false default: Default::default + - name: java_catalog_props + field_type: HashMap + required: false KafkaConfig: fields: - name: properties.bootstrap.server field_type: String required: true alias: kafka.brokers - - name: broker.rewrite.endpoints - field_type: HashMap - required: false - name: topic field_type: String required: true @@ -252,6 +255,9 @@ KafkaConfig: field_type: String comments: Client identifier required: false + - name: properties.enable.ssl.certificate.verification + field_type: bool + required: false - name: properties.allow.auto.create.topics field_type: bool comments: Allow automatic topic creation on the broker when subscribing to or assigning non-existent topics. @@ -302,6 +308,10 @@ KafkaConfig: comments: The maximum number of unacknowledged requests the client will send on a single connection before blocking. required: false default: '5' + - name: broker.rewrite.endpoints + field_type: HashMap + comments: This is generated from `private_link_targets` and `private_link_endpoint` in frontend, instead of given by users. + required: false KinesisSinkConfig: fields: - name: stream @@ -449,3 +459,41 @@ RedisConfig: - name: redis.url field_type: String required: true +StarrocksConfig: + fields: + - name: starrocks.host + field_type: String + comments: The StarRocks host address. + required: true + - name: starrocks.mysqlport + field_type: String + comments: The port to the MySQL server of StarRocks FE. + required: true + alias: starrocks.query_port + - name: starrocks.httpport + field_type: String + comments: The port to the HTTP server of StarRocks FE. + required: true + alias: starrocks.http_port + - name: starrocks.user + field_type: String + comments: The user name used to access the StarRocks database. + required: true + - name: starrocks.password + field_type: String + comments: The password associated with the user. + required: true + - name: starrocks.database + field_type: String + comments: The StarRocks database where the target table is located + required: true + - name: starrocks.table + field_type: String + comments: The StarRocks table you want to sink data to. + required: true + - name: starrocks.partial_update + field_type: String + required: false + - name: r#type + field_type: String + required: true diff --git a/src/connector/with_options_source.yaml b/src/connector/with_options_source.yaml index 84a125813733e..dec3cf6a8941a 100644 --- a/src/connector/with_options_source.yaml +++ b/src/connector/with_options_source.yaml @@ -33,6 +33,35 @@ GcsProperties: field_type: String required: false default: Default::default +IcebergProperties: + fields: + - name: catalog.type + field_type: String + required: true + - name: s3.region + field_type: String + required: true + - name: s3.endpoint + field_type: String + required: false + default: Default::default + - name: s3.access.key + field_type: String + required: false + default: Default::default + - name: s3.secret.key + field_type: String + required: false + default: Default::default + - name: warehouse.path + field_type: String + required: true + - name: database.name + field_type: String + required: true + - name: table.name + field_type: String + required: true KafkaProperties: fields: - name: bytes.per.second @@ -61,9 +90,6 @@ KafkaProperties: field_type: String required: true alias: kafka.brokers - - name: broker.rewrite.endpoints - field_type: HashMap - required: false - name: topic field_type: String required: true @@ -146,6 +172,9 @@ KafkaProperties: field_type: String comments: Client identifier required: false + - name: properties.enable.ssl.certificate.verification + field_type: bool + required: false - name: properties.queued.min.messages field_type: usize comments: Minimum number of messages per topic+partition librdkafka tries to maintain in the local consumer queue. @@ -169,6 +198,10 @@ KafkaProperties: field_type: bool comments: 'Automatically and periodically commit offsets in the background. Note: setting this to false does not prevent the consumer from fetching previously committed start offsets. To circumvent this behaviour set specific start offsets per partition in the call to assign(). default: true' required: false + - name: broker.rewrite.endpoints + field_type: HashMap + comments: This is generated from `private_link_targets` and `private_link_endpoint` in frontend, instead of given by users. + required: false KinesisProperties: fields: - name: scan.startup.mode diff --git a/src/ctl/Cargo.toml b/src/ctl/Cargo.toml index 8b3608cf5076e..a5c67912f4913 100644 --- a/src/ctl/Cargo.toml +++ b/src/ctl/Cargo.toml @@ -21,8 +21,10 @@ clap = { version = "4", features = ["derive"] } comfy-table = "7" etcd-client = { workspace = true } futures = { version = "0.3", default-features = false, features = ["alloc"] } -inquire = "0.6.2" +hex = "0.4" +inquire = "0.7.0" itertools = "0.12" +prost = { workspace = true } regex = "1.10.0" risingwave_common = { workspace = true } risingwave_connector = { workspace = true } diff --git a/src/ctl/src/cmd_impl/hummock/sst_dump.rs b/src/ctl/src/cmd_impl/hummock/sst_dump.rs index b3f3a33af82ac..0fc65054b51e4 100644 --- a/src/ctl/src/cmd_impl/hummock/sst_dump.rs +++ b/src/ctl/src/cmd_impl/hummock/sst_dump.rs @@ -203,7 +203,7 @@ pub async fn sst_dump_via_sstable_store( let sstable_cache = sstable_store .sstable(&sstable_info, &mut StoreLocalStatistic::default()) .await?; - let sstable = sstable_cache.value().as_ref(); + let sstable = sstable_cache.as_ref(); let sstable_meta = &sstable.meta; let smallest_key = FullKey::decode(&sstable_meta.smallest_key); let largest_key = FullKey::decode(&sstable_meta.largest_key); diff --git a/src/ctl/src/cmd_impl/hummock/validate_version.rs b/src/ctl/src/cmd_impl/hummock/validate_version.rs index c09b93e948143..b2ae1c22f66cf 100644 --- a/src/ctl/src/cmd_impl/hummock/validate_version.rs +++ b/src/ctl/src/cmd_impl/hummock/validate_version.rs @@ -12,9 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::cmp::Ordering; + +use chrono::offset::Utc; +use chrono::DateTime; +use itertools::Itertools; +use risingwave_common::util::epoch::Epoch; use risingwave_hummock_sdk::compaction_group::hummock_version_ext; +use risingwave_hummock_sdk::key::{FullKey, UserKey}; +use risingwave_hummock_sdk::version::{HummockVersion, HummockVersionDelta}; +use risingwave_hummock_sdk::{version_archive_dir, HummockSstableObjectId, HummockVersionId}; +use risingwave_object_store::object::ObjectStoreRef; +use risingwave_pb::hummock::group_delta::DeltaType; +use risingwave_pb::hummock::{HummockVersionArchive, SstableInfo}; use risingwave_rpc_client::HummockMetaClient; +use risingwave_storage::hummock::value::HummockValue; +use risingwave_storage::hummock::{Block, BlockHolder, BlockIterator, SstableStoreRef}; +use risingwave_storage::monitor::StoreLocalStatistic; +use crate::common::HummockServiceOpts; use crate::CtlContext; pub async fn validate_version(context: &CtlContext) -> anyhow::Result<()> { @@ -30,3 +46,177 @@ pub async fn validate_version(context: &CtlContext) -> anyhow::Result<()> { Ok(()) } + +async fn get_archive( + archive_id: HummockVersionId, + data_dir: &str, + archive_object_store: ObjectStoreRef, +) -> anyhow::Result { + use prost::Message; + let archive_dir = version_archive_dir(data_dir); + let archive_path = format!("{archive_dir}/{archive_id}"); + let archive_bytes = archive_object_store.read(&archive_path, ..).await?; + let archive: HummockVersionArchive = HummockVersionArchive::decode(archive_bytes)?; + Ok(archive) +} + +pub async fn print_user_key_in_archive( + context: &CtlContext, + archive_ids: Vec, + data_dir: String, + user_key: String, +) -> anyhow::Result<()> { + let user_key_bytes = hex::decode(user_key.clone()).unwrap_or_else(|_| { + panic!("cannot decode user key {} into raw bytes", user_key); + }); + let user_key = UserKey::decode(&user_key_bytes); + println!("user key: {user_key:?}"); + + let hummock_opts = HummockServiceOpts::from_env(Some(data_dir.clone()))?; + let hummock = context.hummock_store(hummock_opts).await?; + let sstable_store = hummock.sstable_store(); + let archive_object_store = sstable_store.store(); + for archive_id in archive_ids.into_iter().sorted() { + println!("search archive {archive_id}"); + let archive = get_archive(archive_id, &data_dir, archive_object_store.clone()).await?; + let mut base_version = + HummockVersion::from_persisted_protobuf(archive.version.as_ref().unwrap()); + print_user_key_in_version(sstable_store.clone(), &base_version, &user_key).await?; + for delta in &archive.version_deltas { + base_version.apply_version_delta(&HummockVersionDelta::from_persisted_protobuf(delta)); + print_user_key_in_version(sstable_store.clone(), &base_version, &user_key).await?; + } + } + Ok(()) +} + +async fn print_user_key_in_version( + sstable_store: SstableStoreRef, + version: &HummockVersion, + target_key: &UserKey<&[u8]>, +) -> anyhow::Result<()> { + println!("print key {:?} in version {}", target_key, version.id); + for cg in version.levels.values() { + for level in cg + .l0 + .as_ref() + .unwrap() + .sub_levels + .iter() + .rev() + .chain(cg.levels.iter()) + { + for sstable_info in &level.table_infos { + use risingwave_hummock_sdk::key_range::KeyRange; + let key_range: KeyRange = sstable_info.key_range.as_ref().unwrap().into(); + let left_user_key = FullKey::decode(&key_range.left); + let right_user_key = FullKey::decode(&key_range.right); + if left_user_key.user_key > *target_key || *target_key > right_user_key.user_key { + continue; + } + print_user_key_in_sst(sstable_store.clone(), sstable_info, target_key).await?; + } + } + } + Ok(()) +} + +async fn print_user_key_in_sst( + sstable_store: SstableStoreRef, + sst: &SstableInfo, + user_key: &UserKey<&[u8]>, +) -> anyhow::Result<()> { + // The implementation is mostly the same as `sst_dump`, with additional filter by `user_key`. + let mut dummy = StoreLocalStatistic::default(); + let sst_metadata = sstable_store.sstable(sst, &mut dummy).await?; + dummy.ignore(); + let data_path = sstable_store.get_sst_data_path(sst_metadata.id); + let mut is_first = true; + for block_meta in &sst_metadata.meta.block_metas { + let range = + block_meta.offset as usize..block_meta.offset as usize + block_meta.len as usize; + let block_data = sstable_store.store().read(&data_path, range).await?; + let block = Box::new(Block::decode(block_data, block_meta.uncompressed_size as _).unwrap()); + let holder = BlockHolder::from_owned_block(block); + let mut block_iter = BlockIterator::new(holder); + block_iter.seek_to_first(); + while block_iter.is_valid() { + let full_key = block_iter.key(); + if full_key.user_key.cmp(user_key) != Ordering::Equal { + block_iter.next(); + continue; + } + let full_val = block_iter.value(); + let hummock_val = HummockValue::from_slice(full_val)?; + let epoch = Epoch::from(full_key.epoch_with_gap.pure_epoch()); + let date_time = DateTime::::from(epoch.as_system_time()); + if is_first { + is_first = false; + println!("\t\tSST id: {}, object id: {}", sst.sst_id, sst.object_id); + } + println!("\t\t key: {:?}, len={}", full_key, full_key.encoded_len()); + println!( + "\t\t value: {:?}, len={}", + hummock_val, + hummock_val.encoded_len() + ); + println!( + "\t\t epoch: {} offset = {} ({})", + epoch, + full_key.epoch_with_gap.offset(), + date_time + ); + println!(); + block_iter.next(); + } + } + Ok(()) +} + +pub async fn print_version_delta_in_archive( + context: &CtlContext, + archive_ids: Vec, + data_dir: String, + sst_id: HummockSstableObjectId, +) -> anyhow::Result<()> { + let hummock_opts = HummockServiceOpts::from_env(Some(data_dir.clone()))?; + let hummock = context.hummock_store(hummock_opts).await?; + let sstable_store = hummock.sstable_store(); + let archive_object_store = sstable_store.store(); + for archive_id in archive_ids.into_iter().sorted() { + println!("search archive {archive_id}"); + let archive = get_archive(archive_id, &data_dir, archive_object_store.clone()).await?; + for delta in &archive.version_deltas { + let mut is_first = true; + for (cg_id, deltas) in &delta.group_deltas { + for d in &deltas.group_deltas { + let d = d.delta_type.as_ref().unwrap(); + if match_delta(d, sst_id) { + if is_first { + is_first = false; + println!("delta: id {}, prev_id {}, max_committed_epoch {}, trivial_move {}, safe_epoch {}", delta.id, delta.prev_id, delta.max_committed_epoch, delta.trivial_move, delta.safe_epoch); + } + println!("compaction group id {cg_id}"); + print_delta(d); + } + } + } + } + } + Ok(()) +} + +fn match_delta(delta: &DeltaType, sst_id: HummockSstableObjectId) -> bool { + let DeltaType::IntraLevel(delta) = delta else { + return false; + }; + delta + .inserted_table_infos + .iter() + .any(|sst| sst.sst_id == sst_id) + || delta.removed_table_ids.iter().any(|sst| *sst == sst_id) +} + +fn print_delta(delta: &DeltaType) { + println!("{:?}", delta); +} diff --git a/src/ctl/src/cmd_impl/profile.rs b/src/ctl/src/cmd_impl/profile.rs index 55e996cca5e7c..edb48df5aeb39 100644 --- a/src/ctl/src/cmd_impl/profile.rs +++ b/src/ctl/src/cmd_impl/profile.rs @@ -35,7 +35,11 @@ pub async fn cpu_profile(context: &CtlContext, sleep_s: u64) -> anyhow::Result<( let clients = ComputeClientPool::default(); - let profile_root_path = PathBuf::from(&std::env::var("PREFIX_PROFILING")?); + let profile_root_path = std::env::var("PREFIX_PROFILING").unwrap_or_else(|_| { + tracing::info!("PREFIX_PROFILING is not set, using current directory"); + "./".to_string() + }); + let profile_root_path = PathBuf::from(&profile_root_path); let dir_name = Local::now().format("%Y-%m-%d-%H-%M-%S").to_string(); let dir_path = profile_root_path.join(dir_name); create_dir_all(&dir_path).await?; diff --git a/src/ctl/src/common/hummock_service.rs b/src/ctl/src/common/hummock_service.rs index 7eb8af52b51ae..b88dd265d382b 100644 --- a/src/ctl/src/common/hummock_service.rs +++ b/src/ctl/src/common/hummock_service.rs @@ -17,14 +17,14 @@ use std::sync::Arc; use std::time::Duration; use anyhow::{anyhow, bail, Result}; -use risingwave_common::config::ObjectStoreConfig; +use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_object_store::object::build_remote_object_store; use risingwave_rpc_client::MetaClient; use risingwave_storage::hummock::hummock_meta_client::MonitoredHummockMetaClient; -use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore}; +use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore, SstableStoreConfig}; use risingwave_storage::monitor::{ - CompactorMetrics, HummockMetrics, HummockStateStoreMetrics, MonitoredStateStore, - MonitoredStorageMetrics, ObjectStoreMetrics, + global_hummock_state_store_metrics, CompactorMetrics, HummockMetrics, HummockStateStoreMetrics, + MonitoredStateStore, MonitoredStorageMetrics, ObjectStoreMetrics, }; use risingwave_storage::opts::StorageOpts; use risingwave_storage::{StateStore, StateStoreImpl}; @@ -162,17 +162,20 @@ impl HummockServiceOpts { let opts = self.get_storage_opts(); - Ok(Arc::new(SstableStore::new( - Arc::new(object_store), - opts.data_directory, - opts.block_cache_capacity_mb * (1 << 20), - opts.meta_cache_capacity_mb * (1 << 20), - 0, - opts.block_cache_capacity_mb * (1 << 20), - opts.max_prefetch_block_number, - FileCache::none(), - FileCache::none(), - None, - ))) + Ok(Arc::new(SstableStore::new(SstableStoreConfig { + store: Arc::new(object_store), + path: opts.data_directory, + block_cache_capacity: opts.block_cache_capacity_mb * (1 << 20), + meta_cache_capacity: opts.meta_cache_capacity_mb * (1 << 20), + high_priority_ratio: 0, + prefetch_buffer_capacity: opts.block_cache_capacity_mb * (1 << 20), + max_prefetch_block_number: opts.max_prefetch_block_number, + data_file_cache: FileCache::none(), + meta_file_cache: FileCache::none(), + recent_filter: None, + state_store_metrics: Arc::new(global_hummock_state_store_metrics( + MetricLevel::Disabled, + )), + }))) } } diff --git a/src/ctl/src/lib.rs b/src/ctl/src/lib.rs index 936a24a8d4bb6..9cdc99c0d3156 100644 --- a/src/ctl/src/lib.rs +++ b/src/ctl/src/lib.rs @@ -22,6 +22,7 @@ use cmd_impl::hummock::SstDumpArgs; use risingwave_hummock_sdk::HummockEpoch; use risingwave_meta::backup_restore::RestoreOpts; use risingwave_pb::meta::update_worker_node_schedulability_request::Schedulability; +use thiserror_ext::AsReport; use crate::cmd_impl::hummock::{ build_compaction_config_vec, list_pinned_snapshots, list_pinned_versions, @@ -266,11 +267,32 @@ enum HummockCommands { ValidateVersion, /// Rebuild table stats RebuildTableStats, - CancelCompactTask { #[clap(short, long)] task_id: u64, }, + PrintUserKeyInArchive { + /// The ident of the archive file in object store. It's also the first Hummock version id of this archive. + #[clap(long, value_delimiter = ',')] + archive_ids: Vec, + /// The data directory of Hummock storage, where SSTable objects can be found. + #[clap(long)] + data_dir: String, + /// KVs that are matched with the user key are printed. + #[clap(long)] + user_key: String, + }, + PrintVersionDeltaInArchive { + /// The ident of the archive file in object store. It's also the first Hummock version id of this archive. + #[clap(long, value_delimiter = ',')] + archive_ids: Vec, + /// The data directory of Hummock storage, where SSTable objects can be found. + #[clap(long)] + data_dir: String, + /// Version deltas that are related to the SST id are printed. + #[clap(long)] + sst_id: u64, + }, } #[derive(Subcommand)] @@ -529,14 +551,28 @@ pub enum ProfileCommands { }, } -pub async fn start(opts: CliOpts) -> Result<()> { +/// Start `risectl` with the given options. +/// Log and abort the process if any error occurs. +/// +/// Note: use [`start_fallible`] if you want to call functionalities of `risectl` +/// in an embedded manner. +pub async fn start(opts: CliOpts) { + if let Err(e) = start_fallible(opts).await { + eprintln!("Error: {:#?}", e.as_report()); // pretty with backtrace + std::process::exit(1); + } +} + +/// Start `risectl` with the given options. +/// Return `Err` if any error occurs. +pub async fn start_fallible(opts: CliOpts) -> Result<()> { let context = CtlContext::default(); let result = start_impl(opts, &context).await; context.try_close().await; result } -pub async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> { +async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> { match opts.command { Commands::Compute(ComputeCommands::ShowConfig { host }) => { cmd_impl::compute::show_config(&host).await? @@ -667,6 +703,27 @@ pub async fn start_impl(opts: CliOpts, context: &CtlContext) -> Result<()> { Commands::Hummock(HummockCommands::CancelCompactTask { task_id }) => { cmd_impl::hummock::cancel_compact_task(context, task_id).await?; } + Commands::Hummock(HummockCommands::PrintVersionDeltaInArchive { + archive_ids, + data_dir, + sst_id, + }) => { + cmd_impl::hummock::print_version_delta_in_archive( + context, + archive_ids, + data_dir, + sst_id, + ) + .await?; + } + Commands::Hummock(HummockCommands::PrintUserKeyInArchive { + archive_ids, + data_dir, + user_key, + }) => { + cmd_impl::hummock::print_user_key_in_archive(context, archive_ids, data_dir, user_key) + .await?; + } Commands::Table(TableCommands::Scan { mv_name, data_dir }) => { cmd_impl::table::scan(context, mv_name, data_dir).await? } diff --git a/src/source/Cargo.toml b/src/dml/Cargo.toml similarity index 90% rename from src/source/Cargo.toml rename to src/dml/Cargo.toml index 735ca5f10d9b6..c429ad9238649 100644 --- a/src/source/Cargo.toml +++ b/src/dml/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "risingwave_source" +name = "risingwave_dml" version = { workspace = true } edition = { workspace = true } homepage = { workspace = true } @@ -14,7 +14,6 @@ ignored = ["workspace-hack"] normal = ["workspace-hack"] [dependencies] -anyhow = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } futures-async-stream = { workspace = true } itertools = "0.12" @@ -23,6 +22,9 @@ rand = "0.8" risingwave_common = { workspace = true } risingwave_connector = { workspace = true } risingwave_pb = { workspace = true } +rw_futures_util = { workspace = true } +thiserror = "1" +thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = ["rt", "rt-multi-thread", "sync", "macros", "time", "signal", "fs"] } tracing = { version = "0.1" } @@ -35,9 +37,5 @@ criterion = { workspace = true, features = ["async_tokio"] } paste = "1" tempfile = "3" -[[bench]] -name = "json_parser" -harness = false - [lints] workspace = true diff --git a/src/source/src/dml_manager.rs b/src/dml/src/dml_manager.rs similarity index 96% rename from src/source/src/dml_manager.rs rename to src/dml/src/dml_manager.rs index b4b03f9798c56..f6611b6c94c21 100644 --- a/src/source/src/dml_manager.rs +++ b/src/dml/src/dml_manager.rs @@ -17,14 +17,12 @@ use std::collections::hash_map::Entry; use std::collections::HashMap; use std::sync::{Arc, Weak}; -use anyhow::Context; use parking_lot::RwLock; -use risingwave_common::bail; use risingwave_common::catalog::{ColumnDesc, TableId, TableVersionId}; -use risingwave_common::error::Result; use risingwave_common::transaction::transaction_id::{TxnId, TxnIdGenerator}; use risingwave_common::util::worker_util::WorkerNodeId; +use crate::error::{DmlError, Result}; use crate::{TableDmlHandle, TableDmlHandleRef}; pub type DmlManagerRef = Arc; @@ -112,9 +110,7 @@ impl DmlManager { "dml handler registers with same version but different schema" ) }) - .with_context(|| { - format!("fail to register reader for table with key `{table_id:?}`") - })?, + .expect("the first dml executor is gone"), // this should never happen // A new version of the table is activated, overwrite the old reader. Ordering::Greater => new_handle!(o), @@ -139,7 +135,7 @@ impl DmlManager { // A new version of the table is activated, but the DML request is still on // the old version. Ordering::Less => { - bail!("schema changed for table `{table_id:?}`, please retry later") + return Err(DmlError::SchemaChanged); } // Write the chunk of correct version to the table. @@ -155,7 +151,7 @@ impl DmlManager { None => None, } } - .with_context(|| format!("no reader for dml in table `{table_id:?}`"))?; + .ok_or(DmlError::NoReader)?; Ok(table_dml_handle) } diff --git a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/CdcSourceMode.java b/src/dml/src/error.rs similarity index 61% rename from java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/CdcSourceMode.java rename to src/dml/src/error.rs index 209e034d0fa72..d8c4ea41a03ce 100644 --- a/java/connector-node/risingwave-connector-service/src/main/java/com/risingwave/connector/source/common/CdcSourceMode.java +++ b/src/dml/src/error.rs @@ -12,11 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -package com.risingwave.connector.source.common; +/// The error type for DML operations. +#[derive(thiserror::Error, Debug)] +pub enum DmlError { + #[error("table schema has changed, please try again later")] + SchemaChanged, -public enum CdcSourceMode { - // The source is dedicated to a single CDC table - SINGLE_MODE, - // The source is sharing by multiple CDC tables - SHARING_MODE, + #[error("no available table reader in streaming executors")] + NoReader, + + #[error("table reader closed")] + ReaderClosed, } + +pub type Result = std::result::Result; diff --git a/src/source/src/lib.rs b/src/dml/src/lib.rs similarity index 86% rename from src/source/src/lib.rs rename to src/dml/src/lib.rs index 8851584680d9f..a15a4dfb3fba9 100644 --- a/src/source/src/lib.rs +++ b/src/dml/src/lib.rs @@ -20,15 +20,11 @@ #![feature(type_alias_impl_trait)] #![feature(box_patterns)] #![feature(stmt_expr_attributes)] +#![feature(error_generic_member_access)] pub use table::*; pub mod dml_manager; - -mod common; -pub mod connector_source; -pub mod source_desc; -pub use source_desc::test_utils as connector_test_utils; -pub mod fs_connector_source; +pub mod error; mod table; mod txn_channel; diff --git a/src/source/src/table.rs b/src/dml/src/table.rs similarity index 93% rename from src/source/src/table.rs rename to src/dml/src/table.rs index ba0292e4f6caf..4981ebce2e8a3 100644 --- a/src/source/src/table.rs +++ b/src/dml/src/table.rs @@ -14,17 +14,15 @@ use std::sync::Arc; -use anyhow::{anyhow, Context}; use futures_async_stream::try_stream; use parking_lot::RwLock; use risingwave_common::array::StreamChunk; use risingwave_common::catalog::ColumnDesc; -use risingwave_common::error::{Result, RwError}; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::transaction::transaction_message::TxnMsg; -use risingwave_connector::source::StreamChunkWithState; use tokio::sync::oneshot; +use crate::error::{DmlError, Result}; use crate::txn_channel::{txn_channel, Receiver, Sender}; pub type TableDmlHandleRef = Arc; @@ -89,9 +87,7 @@ impl TableDmlHandle { loop { let guard = self.core.read(); if guard.changes_txs.is_empty() { - return Err(RwError::from(anyhow!( - "no available table reader in streaming source executors" - ))); + return Err(DmlError::NoReader); } let len = guard.changes_txs.len(); // Use session id instead of txn_id to choose channel so that we can preserve transaction order in the same session. @@ -99,7 +95,7 @@ impl TableDmlHandle { let sender = guard .changes_txs .get((session_id % len as u32) as usize) - .context("no available table reader in streaming source executors")? + .unwrap() .clone(); drop(guard); @@ -183,7 +179,8 @@ impl WriteHandle { pub async fn write_chunk(&self, chunk: StreamChunk) -> Result<()> { assert_eq!(self.txn_state, TxnState::Begin); // Ignore the notifier. - self.write_txn_data_msg(TxnMsg::Data(self.txn_id, chunk)) + let _notifier = self + .write_txn_data_msg(TxnMsg::Data(self.txn_id, chunk)) .await?; Ok(()) } @@ -192,9 +189,8 @@ impl WriteHandle { assert_eq!(self.txn_state, TxnState::Begin); self.txn_state = TxnState::Committed; // Await the notifier. - self.write_txn_control_msg(TxnMsg::End(self.txn_id))? - .await - .context("failed to wait the end message")?; + let notifier = self.write_txn_control_msg(TxnMsg::End(self.txn_id))?; + notifier.await.map_err(|_| DmlError::ReaderClosed)?; Ok(()) } @@ -221,7 +217,7 @@ impl WriteHandle { // It's possible that the source executor is scaled in or migrated, so the channel // is closed. To guarantee the transactional atomicity, bail out. - Err(_) => Err(RwError::from("write txn_msg channel closed".to_string())), + Err(_) => Err(DmlError::ReaderClosed), } } @@ -235,7 +231,7 @@ impl WriteHandle { // It's possible that the source executor is scaled in or migrated, so the channel // is closed. To guarantee the transactional atomicity, bail out. - Err(_) => Err(RwError::from("write txn_msg channel closed".to_string())), + Err(_) => Err(DmlError::ReaderClosed), } } } @@ -251,7 +247,7 @@ pub struct TableStreamReader { } impl TableStreamReader { - #[try_stream(boxed, ok = StreamChunkWithState, error = RwError)] + #[try_stream(boxed, ok = StreamChunk, error = DmlError)] pub async fn into_data_stream_for_test(mut self) { while let Some((txn_msg, notifier)) = self.rx.recv().await { // Notify about that we've taken the chunk. @@ -261,13 +257,13 @@ impl TableStreamReader { } TxnMsg::Data(_, chunk) => { _ = notifier.send(chunk.cardinality()); - yield chunk.into(); + yield chunk; } } } } - #[try_stream(boxed, ok = TxnMsg, error = RwError)] + #[try_stream(boxed, ok = TxnMsg, error = DmlError)] pub async fn into_stream(mut self) { while let Some((txn_msg, notifier)) = self.rx.recv().await { // Notify about that we've taken the chunk. diff --git a/src/source/src/txn_channel.rs b/src/dml/src/txn_channel.rs similarity index 100% rename from src/source/src/txn_channel.rs rename to src/dml/src/txn_channel.rs diff --git a/src/error/Cargo.toml b/src/error/Cargo.toml index 13bb50a371853..4a99711db6c41 100644 --- a/src/error/Cargo.toml +++ b/src/error/Cargo.toml @@ -8,6 +8,7 @@ license = { workspace = true } repository = { workspace = true } [dependencies] +anyhow = "1" bincode = "1" bytes = "1" easy-ext = "1" diff --git a/src/error/src/anyhow.rs b/src/error/src/anyhow.rs new file mode 100644 index 0000000000000..0acadddd88fa3 --- /dev/null +++ b/src/error/src/anyhow.rs @@ -0,0 +1,146 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Define a newtype wrapper around [`anyhow::Error`]. +/// +/// # Usage +/// +/// ```ignore +/// def_anyhow_newtype! { +/// /// Documentation for the newtype. +/// #[derive(..)] +/// pub MyError, +/// +/// // Default context messages for each source error type goes below. +/// mysql::Error => "failed to interact with MySQL", +/// postgres::Error => "failed to interact with PostgreSQL", +/// opendal::Error => transparent, // if it's believed to be self-explanatory +/// // and any context is not necessary +/// } +/// ``` +/// +/// # Construction +/// +/// Unlike [`anyhow::Error`], the newtype **CANNOT** be converted from any error +/// types implicitly. Instead, it can only be converted from [`anyhow::Error`] +/// by default. +/// +/// - Users are encouraged to use [`anyhow::Context`] to attach detailed +/// information to the source error and make it an [`anyhow::Error`] before +/// converting it to the newtype. +/// +/// - Otherwise, specify the default context for each source error type as shown +/// in the example above, which will be expanded into a `From` implementation +/// from the source error type to the newtype. This should **NOT** be preferred +/// in most cases since it's less informative than the ad-hoc context provided +/// with [`anyhow::Context`] at the call site, but it could still be useful +/// during refactoring, or if the source error type is believed to be +/// self-explanatory. +/// +/// To construct a new error from scratch, one can still use macros like +/// `anyhow::anyhow!` or `risingwave_common::bail!`. Since `bail!` and `?` +/// already imply an `into()` call, developers do not need to care about the +/// type conversion most of the time. +/// +/// ## Example +/// +/// ```ignore +/// fn read_offset_from_mysql() -> Result { +/// .. +/// } +/// fn parse_offset(offset: &str) -> Result { +/// .. +/// } +/// +/// fn work() -> Result<(), MyError> { +/// // `mysql::Error` can be converted to `MyError` implicitly with `?` +/// // as the default context is provided in the definition. +/// let offset = read_offset_from_mysql()?; +/// +/// // Instead, `ParseIntError` cannot be directly converted to `MyError` +/// // with `?`, so the caller must attach context explicitly. +/// // +/// // This makes sense as the semantics of the integer ("offset") holds +/// // important information and are not implied by the error type itself. +/// let offset = parse_offset(&offset).context("failed to parse offset")?; +/// +/// if offset < 0 { +/// // Construct a new error with `bail!` macro. +/// bail!("offset `{}` must be non-negative", offset); +/// } +/// } +/// ``` +/// +/// # Discussion +/// +/// - What's the purpose of the newtype? +/// * It is to provide extra type information for errors, which makes it +/// clearer to identify which module or crate the error comes from when +/// it is passed around. +/// * It enforces the developer to attach context (explicitly or by default) +/// when doing type conversion, which makes the error more informative. +/// +/// - Is the effect essentially the same as `thiserror`? +/// * Yes, but we're here intentionally making the error type less actionable +/// to make it informative with no fear. +/// * To elaborate, consider the following `thiserror` example: +/// ```ignore +/// #[derive(thiserror::Error, Debug)] +/// pub enum MyError { +/// #[error("failed to interact with MySQL")] +/// MySql(#[from] mysql::Error), +/// #[error(transparent)] +/// Other(#[from] anyhow::Error), +/// } +/// ``` +/// This gives the caller an illusion that all errors related to MySQL are +/// under the `MySql` variant, which is not true as one could attach context +/// to an `mysql::Error` with [`anyhow::Context`] and make it go into the +/// `Other` variant. +/// +/// By doing type erasure with `anyhow`, we're making it clear that the +/// error is not actionable so such confusion is avoided. +#[macro_export] +macro_rules! def_anyhow_newtype { + (@from $error:ident transparent) => { + Self(::anyhow::Error::new($error)) + }; + (@from $error:ident $context:literal) => { + Self(::anyhow::Error::new($error).context($context)) + }; + + ( + $(#[$attr:meta])* $vis:vis $name:ident + $(, $from:ty => $context:tt)* $(,)? + ) => { + #[derive(::thiserror::Error, ::std::fmt::Debug)] + #[error(transparent)] + $(#[$attr])* $vis struct $name(#[from] #[backtrace] ::anyhow::Error); + + impl $name { + /// Unwrap the newtype to get the inner [`anyhow::Error`]. + pub fn into_inner(self) -> ::anyhow::Error { + self.0 + } + } + + $( + impl From<$from> for $name { + fn from(error: $from) -> Self { + def_anyhow_newtype!(@from error $context) + } + } + )* + }; +} diff --git a/src/error/src/lib.rs b/src/error/src/lib.rs index ccfec0cfbcc19..f7a2611b84a65 100644 --- a/src/error/src/lib.rs +++ b/src/error/src/lib.rs @@ -21,4 +21,8 @@ #![feature(register_tool)] #![register_tool(rw)] +pub mod anyhow; pub mod tonic; + +// Re-export the `thiserror-ext` crate. +pub use thiserror_ext::*; diff --git a/src/expr/core/Cargo.toml b/src/expr/core/Cargo.toml index ff1910e9a0182..03ed84ac6a216 100644 --- a/src/expr/core/Cargo.toml +++ b/src/expr/core/Cargo.toml @@ -15,10 +15,15 @@ ignored = ["workspace-hack", "ctor"] [package.metadata.cargo-udeps.ignore] normal = ["workspace-hack", "ctor"] +[features] +embedded-python-udf = ["arrow-udf-python"] + [dependencies] anyhow = "1" arrow-array = { workspace = true } arrow-schema = { workspace = true } +arrow-udf-js = { workspace = true } +arrow-udf-python = { workspace = true, optional = true } arrow-udf-wasm = { workspace = true } async-trait = "0.1" auto_impl = "1" @@ -28,21 +33,23 @@ chrono = { version = "0.4", default-features = false, features = [ "clock", "std", ] } -ctor = "0.2" downcast-rs = "1.2" easy-ext = "1" +educe = "0.5" either = "1" enum-as-inner = "0.6" futures-async-stream = { workspace = true } futures-util = "0.3" itertools = "0.12" -moka = { version = "0.12", features = ["future"] } +linkme = { version = "0.3", features = ["used_linker"] } +md5 = "0.7" +moka = { version = "0.12", features = ["sync"] } num-traits = "0.2" -parse-display = "0.8" +openssl = { version = "0.10", features = ["vendored"] } +parse-display = "0.9" paste = "1" risingwave_common = { workspace = true } risingwave_expr_macro = { path = "../macro" } -risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_udf = { workspace = true } smallvec = "1" @@ -54,6 +61,7 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [ "macros", ] } tracing = "0.1" +zstd = { version = "0.13", default-features = false } [target.'cfg(not(madsim))'.dependencies] workspace-hack = { path = "../../workspace-hack" } diff --git a/src/expr/core/src/codegen.rs b/src/expr/core/src/codegen.rs index a22f5f069dec6..c9022e40aec91 100644 --- a/src/expr/core/src/codegen.rs +++ b/src/expr/core/src/codegen.rs @@ -13,7 +13,7 @@ // limitations under the License. pub use async_trait::async_trait; -pub use ctor::ctor; pub use futures_async_stream::try_stream; pub use futures_util::stream::BoxStream; pub use itertools::multizip; +pub use linkme; diff --git a/src/expr/core/src/error.rs b/src/expr/core/src/error.rs index a9646c1d77035..6688824093d2d 100644 --- a/src/expr/core/src/error.rs +++ b/src/expr/core/src/error.rs @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fmt::Display; +use std::fmt::{Debug, Display}; use risingwave_common::array::{ArrayError, ArrayRef}; -use risingwave_common::error::{ErrorCode, RwError}; use risingwave_common::types::DataType; use risingwave_pb::PbFieldNotFound; use thiserror::Error; @@ -117,16 +116,27 @@ pub enum ExprError { #[error("invalid state: {0}")] InvalidState(String), + + #[error("error in cryptography: {0}")] + Cryptography(Box), } -static_assertions::const_assert_eq!(std::mem::size_of::(), 40); +#[derive(Debug)] +pub enum CryptographyStage { + Encrypt, + Decrypt, +} -impl From for RwError { - fn from(s: ExprError) -> Self { - ErrorCode::ExprError(Box::new(s)).into() - } +#[derive(Debug, Error)] +#[error("{stage:?} stage, reason: {reason}")] +pub struct CryptographyError { + pub stage: CryptographyStage, + #[source] + pub reason: openssl::error::ErrorStack, } +static_assertions::const_assert_eq!(std::mem::size_of::(), 40); + impl From for ExprError { fn from(e: chrono::ParseError) -> Self { Self::Parse(e.to_report_string().into()) @@ -156,7 +166,7 @@ impl MultiExprError { impl Display for MultiExprError { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { for (i, e) in self.0.iter().enumerate() { - writeln!(f, "{i}: {e}")?; + writeln!(f, "{i}: {}", e.as_report())?; } Ok(()) } diff --git a/src/expr/core/src/expr/build.rs b/src/expr/core/src/expr/build.rs index 51e9b87de19fe..91ebe67f479a9 100644 --- a/src/expr/core/src/expr/build.rs +++ b/src/expr/core/src/expr/build.rs @@ -131,6 +131,7 @@ pub(crate) trait Build: Expression + Sized { /// Build the expression `Self` from protobuf for test, where each child is built with /// [`build_from_prost`]. + #[cfg(test)] fn build_for_test(prost: &ExprNode) -> Result { Self::build(prost, build_from_prost) } diff --git a/src/expr/core/src/expr/expr_udf.rs b/src/expr/core/src/expr/expr_udf.rs index 60f04799838ba..a319abe8e5e15 100644 --- a/src/expr/core/src/expr/expr_udf.rs +++ b/src/expr/core/src/expr/expr_udf.rs @@ -20,16 +20,17 @@ use std::time::Duration; use anyhow::Context; use arrow_schema::{Field, Fields, Schema}; +use arrow_udf_js::{CallMode as JsCallMode, Runtime as JsRuntime}; +#[cfg(feature = "embedded-python-udf")] +use arrow_udf_python::{CallMode as PythonCallMode, Runtime as PythonRuntime}; use arrow_udf_wasm::Runtime as WasmRuntime; use await_tree::InstrumentAwait; use cfg_or_panic::cfg_or_panic; -use moka::future::Cache; +use moka::sync::Cache; use risingwave_common::array::{ArrayError, ArrayRef, DataChunk}; -use risingwave_common::config::ObjectStoreConfig; use risingwave_common::row::OwnedRow; use risingwave_common::types::{DataType, Datum}; -use risingwave_object_store::object::build_remote_object_store; -use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; +use risingwave_expr::expr_context::FRAGMENT_ID; use risingwave_pb::expr::ExprNode; use risingwave_udf::ArrowFlightUdfClient; use thiserror_ext::AsReport; @@ -43,6 +44,7 @@ pub struct UserDefinedFunction { children: Vec, arg_types: Vec, return_type: DataType, + #[expect(dead_code)] arg_schema: Arc, imp: UdfImpl, identifier: String, @@ -53,14 +55,19 @@ pub struct UserDefinedFunction { /// On each successful call, the count will be decreased by 1. /// See . disable_retry_count: AtomicU8, + /// Always retry. Overrides `disable_retry_count`. + always_retry_on_network_error: bool, } const INITIAL_RETRY_COUNT: u8 = 16; #[derive(Debug)] -enum UdfImpl { +pub enum UdfImpl { External(Arc), Wasm(Arc), + JavaScript(JsRuntime), + #[cfg(feature = "embedded-python-udf")] + Python(PythonRuntime), } #[async_trait::async_trait] @@ -69,18 +76,22 @@ impl Expression for UserDefinedFunction { self.return_type.clone() } - #[cfg_or_panic(not(madsim))] async fn eval(&self, input: &DataChunk) -> Result { - let vis = input.visibility(); + if input.cardinality() == 0 { + // early return for empty input + let mut builder = self.return_type.create_array_builder(input.capacity()); + builder.append_n_null(input.capacity()); + return Ok(builder.finish().into_ref()); + } let mut columns = Vec::with_capacity(self.children.len()); for child in &self.children { let array = child.eval(input).await?; columns.push(array); } - self.eval_inner(columns, vis).await + let chunk = DataChunk::new(columns, input.visibility().clone()); + self.eval_inner(&chunk).await } - #[cfg_or_panic(not(madsim))] async fn eval_row(&self, input: &OwnedRow) -> Result { let mut columns = Vec::with_capacity(self.children.len()); for child in &self.children { @@ -89,81 +100,77 @@ impl Expression for UserDefinedFunction { } let arg_row = OwnedRow::new(columns); let chunk = DataChunk::from_rows(std::slice::from_ref(&arg_row), &self.arg_types); - let arg_columns = chunk.columns().to_vec(); - let output_array = self.eval_inner(arg_columns, chunk.visibility()).await?; + let output_array = self.eval_inner(&chunk).await?; Ok(output_array.to_datum()) } } impl UserDefinedFunction { - async fn eval_inner( - &self, - columns: Vec, - vis: &risingwave_common::buffer::Bitmap, - ) -> Result { - let chunk = DataChunk::new(columns, vis.clone()); - let compacted_chunk = chunk.compact_cow(); - let compacted_columns: Vec = compacted_chunk - .columns() - .iter() - .map(|c| { - c.as_ref() - .try_into() - .expect("failed covert ArrayRef to arrow_array::ArrayRef") - }) - .collect(); - let opts = arrow_array::RecordBatchOptions::default() - .with_row_count(Some(compacted_chunk.capacity())); - let input = arrow_array::RecordBatch::try_new_with_options( - self.arg_schema.clone(), - compacted_columns, - &opts, - ) - .expect("failed to build record batch"); + async fn eval_inner(&self, input: &DataChunk) -> Result { + // this will drop invisible rows + let arrow_input = arrow_array::RecordBatch::try_from(input)?; - let output: arrow_array::RecordBatch = match &self.imp { - UdfImpl::Wasm(runtime) => runtime.call(&self.identifier, &input)?, + let arrow_output: arrow_array::RecordBatch = match &self.imp { + UdfImpl::Wasm(runtime) => runtime.call(&self.identifier, &arrow_input)?, + UdfImpl::JavaScript(runtime) => runtime.call(&self.identifier, &arrow_input)?, + #[cfg(feature = "embedded-python-udf")] + UdfImpl::Python(runtime) => runtime.call(&self.identifier, &arrow_input)?, UdfImpl::External(client) => { + // batch query does not have a fragment_id + let fragment_id = FRAGMENT_ID::try_with(ToOwned::to_owned).unwrap_or(0); + let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed); - let result = if disable_retry_count != 0 { + let result = if self.always_retry_on_network_error { client - .call(&self.identifier, input) + .call_with_always_retry_on_network_error( + &self.identifier, + arrow_input, + fragment_id, + ) .instrument_await(self.span.clone()) .await } else { - client - .call_with_retry(&self.identifier, input) - .instrument_await(self.span.clone()) - .await + let result = if disable_retry_count != 0 { + client + .call(&self.identifier, arrow_input, fragment_id) + .instrument_await(self.span.clone()) + .await + } else { + client + .call_with_retry(&self.identifier, arrow_input, fragment_id) + .instrument_await(self.span.clone()) + .await + }; + let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed); + let connection_error = matches!(&result, Err(e) if e.is_connection_error()); + if connection_error && disable_retry_count != INITIAL_RETRY_COUNT { + // reset count on connection error + self.disable_retry_count + .store(INITIAL_RETRY_COUNT, Ordering::Relaxed); + } else if !connection_error && disable_retry_count != 0 { + // decrease count on success, ignore if exchange failed + _ = self.disable_retry_count.compare_exchange( + disable_retry_count, + disable_retry_count - 1, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + result }; - let disable_retry_count = self.disable_retry_count.load(Ordering::Relaxed); - let connection_error = matches!(&result, Err(e) if e.is_connection_error()); - if connection_error && disable_retry_count != INITIAL_RETRY_COUNT { - // reset count on connection error - self.disable_retry_count - .store(INITIAL_RETRY_COUNT, Ordering::Relaxed); - } else if !connection_error && disable_retry_count != 0 { - // decrease count on success, ignore if exchange failed - _ = self.disable_retry_count.compare_exchange( - disable_retry_count, - disable_retry_count - 1, - Ordering::Relaxed, - Ordering::Relaxed, - ); - } result? } }; - if output.num_rows() != vis.count_ones() { + if arrow_output.num_rows() != input.cardinality() { bail!( "UDF returned {} rows, but expected {}", - output.num_rows(), - vis.len(), + arrow_output.num_rows(), + input.cardinality(), ); } - let data_chunk = DataChunk::try_from(&output)?; - let output = data_chunk.uncompact(vis.clone()); + let output = DataChunk::try_from(&arrow_output)?; + let output = output.uncompact(input.visibility().clone()); let Some(array) = output.columns().first() else { bail!("UDF returned no columns"); @@ -180,7 +187,6 @@ impl UserDefinedFunction { } } -#[cfg_or_panic(not(madsim))] impl Build for UserDefinedFunction { fn build( prost: &ExprNode, @@ -189,16 +195,51 @@ impl Build for UserDefinedFunction { let return_type = DataType::from(prost.get_return_type().unwrap()); let udf = prost.get_rex_node().unwrap().as_udf().unwrap(); + let identifier = udf.get_identifier()?; let imp = match udf.language.as_str() { - "wasm" => { - // Use `block_in_place` as an escape hatch to run async code here in sync context. - // Calling `block_on` directly will panic. - UdfImpl::Wasm(tokio::task::block_in_place(|| { - tokio::runtime::Handle::current() - .block_on(get_or_create_wasm_runtime(&udf.link)) - })?) + #[cfg(not(madsim))] + "wasm" | "rust" => { + let compressed_wasm_binary = udf.get_compressed_binary()?; + let wasm_binary = zstd::stream::decode_all(compressed_wasm_binary.as_slice()) + .context("failed to decompress wasm binary")?; + let runtime = get_or_create_wasm_runtime(&wasm_binary)?; + UdfImpl::Wasm(runtime) + } + "javascript" => { + let mut rt = JsRuntime::new()?; + let body = format!( + "export function {}({}) {{ {} }}", + identifier, + udf.arg_names.join(","), + udf.get_body()? + ); + rt.add_function( + identifier, + arrow_schema::DataType::try_from(&return_type)?, + JsCallMode::CalledOnNullInput, + &body, + )?; + UdfImpl::JavaScript(rt) + } + #[cfg(feature = "embedded-python-udf")] + "python" if udf.body.is_some() => { + let mut rt = PythonRuntime::builder().sandboxed(true).build()?; + let body = udf.get_body()?; + rt.add_function( + identifier, + arrow_schema::DataType::try_from(&return_type)?, + PythonCallMode::CalledOnNullInput, + body, + )?; + UdfImpl::Python(rt) } - _ => UdfImpl::External(get_or_create_flight_client(&udf.link)?), + #[cfg(not(madsim))] + _ => { + let link = udf.get_link()?; + UdfImpl::External(get_or_create_flight_client(link)?) + } + #[cfg(madsim)] + l => panic!("UDF language {l:?} is not supported on madsim"), }; let arg_schema = Arc::new(Schema::new( @@ -222,9 +263,10 @@ impl Build for UserDefinedFunction { return_type, arg_schema, imp, - identifier: udf.identifier.clone(), - span: format!("udf_call({})", udf.identifier).into(), + identifier: identifier.clone(), + span: format!("udf_call({})", identifier).into(), disable_retry_count: AtomicU8::new(0), + always_retry_on_network_error: udf.always_retry_on_network_error, }) } } @@ -251,38 +293,21 @@ pub(crate) fn get_or_create_flight_client(link: &str) -> Result Result> { - static RUNTIMES: LazyLock>> = LazyLock::new(|| { +pub fn get_or_create_wasm_runtime(binary: &[u8]) -> Result> { + static RUNTIMES: LazyLock>> = LazyLock::new(|| { Cache::builder() .time_to_idle(Duration::from_secs(60)) .build() }); - if let Some(runtime) = RUNTIMES.get(link).await { + let md5 = md5::compute(binary); + if let Some(runtime) = RUNTIMES.get(&md5) { return Ok(runtime.clone()); } - // create new runtime - let (wasm_storage_url, object_name) = link - .rsplit_once('/') - .context("invalid link for wasm function")?; - - // load wasm binary from object store - let object_store = build_remote_object_store( - wasm_storage_url, - Arc::new(ObjectStoreMetrics::unused()), - "Wasm Engine", - ObjectStoreConfig::default(), - ) - .await; - let binary = object_store - .read(object_name, ..) - .await - .context("failed to load wasm binary from object storage")?; - - let runtime = Arc::new(arrow_udf_wasm::Runtime::new(&binary)?); - RUNTIMES.insert(link.into(), runtime.clone()).await; + let runtime = Arc::new(arrow_udf_wasm::Runtime::new(binary)?); + RUNTIMES.insert(md5, runtime.clone()); Ok(runtime) } diff --git a/src/expr/core/src/expr_context.rs b/src/expr/core/src/expr_context.rs index 401d5d5e46ee4..27a888118e318 100644 --- a/src/expr/core/src/expr_context.rs +++ b/src/expr/core/src/expr_context.rs @@ -20,6 +20,7 @@ use risingwave_pb::plan_common::ExprContext; // For all execution mode. define_context! { pub TIME_ZONE: String, + pub FRAGMENT_ID: u32, } pub fn capture_expr_context() -> ExprResult { diff --git a/src/expr/core/src/lib.rs b/src/expr/core/src/lib.rs index 93fbb4f5fad6b..fe6d5b9db4d1d 100644 --- a/src/expr/core/src/lib.rs +++ b/src/expr/core/src/lib.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![allow(non_snake_case)] // for `ctor` generated code #![feature(let_chains)] #![feature(lint_reasons)] #![feature(iterator_try_collect)] @@ -20,6 +19,7 @@ #![feature(coroutines)] #![feature(never_type)] #![feature(error_generic_member_access)] +#![feature(used_with_arg)] extern crate self as risingwave_expr; @@ -34,6 +34,6 @@ pub mod sig; pub mod table_function; pub mod window_function; -pub use error::{ContextUnavailable, ExprError, Result}; +pub use error::{ContextUnavailable, CryptographyError, CryptographyStage, ExprError, Result}; pub use risingwave_common::{bail, ensure}; pub use risingwave_expr_macro::*; diff --git a/src/expr/core/src/sig/mod.rs b/src/expr/core/src/sig/mod.rs index 4366b90230cd9..747779f81e990 100644 --- a/src/expr/core/src/sig/mod.rs +++ b/src/expr/core/src/sig/mod.rs @@ -31,12 +31,11 @@ use crate::table_function::BoxedTableFunction; use crate::ExprError; /// The global registry of all function signatures. -pub static FUNCTION_REGISTRY: LazyLock = LazyLock::new(|| unsafe { - // SAFETY: this function is called after all `#[ctor]` functions are called. +pub static FUNCTION_REGISTRY: LazyLock = LazyLock::new(|| { let mut map = FunctionRegistry::default(); - tracing::info!("found {} functions", FUNCTION_REGISTRY_INIT.len()); - for sig in FUNCTION_REGISTRY_INIT.drain(..) { - map.insert(sig); + tracing::info!("found {} functions", FUNCTIONS.len()); + for f in FUNCTIONS { + map.insert(f()); } map }); @@ -459,22 +458,6 @@ pub enum FuncBuilder { Udf, } -/// Register a function into global registry. -/// -/// # Safety -/// -/// This function must be called sequentially. -/// -/// It is designed to be used by `#[function]` macro. -/// Users SHOULD NOT call this function. -#[doc(hidden)] -pub unsafe fn _register(sig: FuncSign) { - FUNCTION_REGISTRY_INIT.push(sig) -} - -/// The global registry of function signatures on initialization. -/// -/// `#[function]` macro will generate a `#[ctor]` function to register the signature into this -/// vector. The calls are guaranteed to be sequential. The vector will be drained and moved into -/// `FUNCTION_REGISTRY` on the first access of `FUNCTION_REGISTRY`. -static mut FUNCTION_REGISTRY_INIT: Vec = Vec::new(); +/// A static distributed slice of functions defined by `#[function]`. +#[linkme::distributed_slice] +pub static FUNCTIONS: [fn() -> FuncSign]; diff --git a/src/expr/core/src/table_function/user_defined.rs b/src/expr/core/src/table_function/user_defined.rs index 83658026ed56d..0d1d726c394c7 100644 --- a/src/expr/core/src/table_function/user_defined.rs +++ b/src/expr/core/src/table_function/user_defined.rs @@ -14,17 +14,20 @@ use std::sync::Arc; +use anyhow::Context; use arrow_array::RecordBatch; use arrow_schema::{Field, Fields, Schema, SchemaRef}; -use arrow_udf_wasm::Runtime as WasmRuntime; +use arrow_udf_js::{CallMode as JsCallMode, Runtime as JsRuntime}; +#[cfg(feature = "embedded-python-udf")] +use arrow_udf_python::{CallMode as PythonCallMode, Runtime as PythonRuntime}; use cfg_or_panic::cfg_or_panic; use futures_util::stream; use risingwave_common::array::{ArrayError, DataChunk, I32Array}; use risingwave_common::bail; -use risingwave_udf::ArrowFlightUdfClient; use thiserror_ext::AsReport; use super::*; +use crate::expr::expr_udf::UdfImpl; #[derive(Debug)] pub struct UserDefinedTableFunction { @@ -38,12 +41,6 @@ pub struct UserDefinedTableFunction { chunk_size: usize, } -#[derive(Debug)] -enum UdfImpl { - External(Arc), - Wasm(Arc), -} - #[async_trait::async_trait] impl TableFunction for UserDefinedTableFunction { fn return_type(&self) -> DataType { @@ -70,6 +67,17 @@ impl UdfImpl { yield res?; } } + UdfImpl::JavaScript(runtime) => { + for res in runtime.call_table_function(identifier, &input, 1024)? { + yield res?; + } + } + #[cfg(feature = "embedded-python-udf")] + UdfImpl::Python(runtime) => { + for res in runtime.call_table_function(identifier, &input, 1024)? { + yield res?; + } + } UdfImpl::Wasm(runtime) => { for res in runtime.call_table_function(identifier, &input)? { yield res?; @@ -177,28 +185,58 @@ pub fn new_user_defined(prost: &PbTableFunction, chunk_size: usize) -> Result()?, )); + let identifier = udtf.get_identifier()?; + let return_type = DataType::from(prost.get_return_type()?); + let client = match udtf.language.as_str() { - "wasm" => { - // Use `block_in_place` as an escape hatch to run async code here in sync context. - // Calling `block_on` directly will panic. - UdfImpl::Wasm(tokio::task::block_in_place(|| { - tokio::runtime::Handle::current().block_on( - crate::expr::expr_udf::get_or_create_wasm_runtime(&udtf.link), - ) - })?) + "wasm" | "rust" => { + let compressed_wasm_binary = udtf.get_compressed_binary()?; + let wasm_binary = zstd::stream::decode_all(compressed_wasm_binary.as_slice()) + .context("failed to decompress wasm binary")?; + let runtime = crate::expr::expr_udf::get_or_create_wasm_runtime(&wasm_binary)?; + UdfImpl::Wasm(runtime) + } + "javascript" => { + let mut rt = JsRuntime::new()?; + let body = format!( + "export function* {}({}) {{ {} }}", + identifier, + udtf.arg_names.join(","), + udtf.get_body()? + ); + rt.add_function( + identifier, + arrow_schema::DataType::try_from(&return_type)?, + JsCallMode::CalledOnNullInput, + &body, + )?; + UdfImpl::JavaScript(rt) + } + #[cfg(feature = "embedded-python-udf")] + "python" if udtf.body.is_some() => { + let mut rt = PythonRuntime::builder().sandboxed(true).build()?; + let body = udtf.get_body()?; + rt.add_function( + identifier, + arrow_schema::DataType::try_from(&return_type)?, + PythonCallMode::CalledOnNullInput, + body, + )?; + UdfImpl::Python(rt) } // connect to UDF service - _ => UdfImpl::External(crate::expr::expr_udf::get_or_create_flight_client( - &udtf.link, - )?), + _ => { + let link = udtf.get_link()?; + UdfImpl::External(crate::expr::expr_udf::get_or_create_flight_client(link)?) + } }; Ok(UserDefinedTableFunction { children: prost.args.iter().map(expr_build_from_prost).try_collect()?, - return_type: prost.return_type.as_ref().expect("no return type").into(), + return_type, arg_schema, client, - identifier: udtf.identifier.clone(), + identifier: identifier.clone(), chunk_size, } .boxed()) diff --git a/src/expr/core/src/window_function/call.rs b/src/expr/core/src/window_function/call.rs index 43545cc2a107a..f5714c64345a1 100644 --- a/src/expr/core/src/window_function/call.rs +++ b/src/expr/core/src/window_function/call.rs @@ -13,17 +13,34 @@ // limitations under the License. use std::fmt::Display; +use std::ops::Deref; +use std::sync::Arc; +use anyhow::Context; +use educe::Educe; use enum_as_inner::EnumAsInner; +use futures_util::FutureExt; use parse_display::Display; -use risingwave_common::bail; -use risingwave_common::types::DataType; -use risingwave_pb::expr::window_frame::{PbBound, PbExclusion}; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{ + DataType, Datum, IsNegative, ScalarImpl, ScalarRefImpl, Sentinelled, ToOwnedDatum, ToText, +}; +use risingwave_common::util::sort_util::{Direction, OrderType}; +use risingwave_common::util::value_encoding::{DatumFromProtoExt, DatumToProtoExt}; +use risingwave_common::{bail, must_match}; +use risingwave_pb::expr::window_frame::{ + PbBound, PbBoundType, PbBounds, PbExclusion, PbRangeFrameBound, PbRangeFrameBounds, + PbRowsFrameBound, PbRowsFrameBounds, +}; use risingwave_pb::expr::{PbWindowFrame, PbWindowFunction}; use FrameBound::{CurrentRow, Following, Preceding, UnboundedFollowing, UnboundedPreceding}; use super::WindowFuncKind; use crate::aggregate::AggArgs; +use crate::expr::{ + build_func, BoxedExpression, Expression, ExpressionBoxExt, InputRefExpression, + LiteralExpression, +}; use crate::Result; #[derive(Debug, Clone)] @@ -63,7 +80,7 @@ impl Display for Frame { } impl Frame { - pub fn rows(start: FrameBound, end: FrameBound) -> Self { + pub fn rows(start: RowsFrameBound, end: RowsFrameBound) -> Self { Self { bounds: FrameBounds::Rows(RowsFrameBounds { start, end }), exclusion: FrameExclusion::default(), @@ -71,8 +88,8 @@ impl Frame { } pub fn rows_with_exclusion( - start: FrameBound, - end: FrameBound, + start: RowsFrameBound, + end: RowsFrameBound, exclusion: FrameExclusion, ) -> Self { Self { @@ -87,11 +104,19 @@ impl Frame { use risingwave_pb::expr::window_frame::PbType; let bounds = match frame.get_type()? { PbType::Unspecified => bail!("unspecified type of `WindowFrame`"), - PbType::Rows => { - let start = FrameBound::from_protobuf(frame.get_start()?)?; - let end = FrameBound::from_protobuf(frame.get_end()?)?; + PbType::RowsLegacy => { + let start = FrameBound::::from_protobuf_legacy(frame.get_start()?)?; + let end = FrameBound::::from_protobuf_legacy(frame.get_end()?)?; FrameBounds::Rows(RowsFrameBounds { start, end }) } + PbType::Rows => { + let bounds = must_match!(frame.get_bounds()?, PbBounds::Rows(bounds) => bounds); + FrameBounds::Rows(RowsFrameBounds::from_protobuf(bounds)?) + } + PbType::Range => { + let bounds = must_match!(frame.get_bounds()?, PbBounds::Range(bounds) => bounds); + FrameBounds::Range(RangeFrameBounds::from_protobuf(bounds)?) + } }; let exclusion = FrameExclusion::from_protobuf(frame.get_exclusion()?)?; Ok(Self { bounds, exclusion }) @@ -101,40 +126,53 @@ impl Frame { use risingwave_pb::expr::window_frame::PbType; let exclusion = self.exclusion.to_protobuf() as _; match &self.bounds { - FrameBounds::Rows(RowsFrameBounds { start, end }) => PbWindowFrame { + #[expect(deprecated)] + FrameBounds::Rows(bounds) => PbWindowFrame { r#type: PbType::Rows as _, - start: Some(start.to_protobuf()), - end: Some(end.to_protobuf()), + start: None, // deprecated + end: None, // deprecated + exclusion, + bounds: Some(PbBounds::Rows(bounds.to_protobuf())), + }, + #[expect(deprecated)] + FrameBounds::Range(bounds) => PbWindowFrame { + r#type: PbType::Range as _, + start: None, // deprecated + end: None, // deprecated exclusion, + bounds: Some(PbBounds::Range(bounds.to_protobuf())), }, } } } -#[derive(Display, Debug, Clone, Eq, PartialEq, Hash)] +#[derive(Display, Debug, Clone, Eq, PartialEq, Hash, EnumAsInner)] #[display("{0}")] pub enum FrameBounds { Rows(RowsFrameBounds), // Groups(GroupsFrameBounds), - // Range(RangeFrameBounds), + Range(RangeFrameBounds), } impl FrameBounds { pub fn validate(&self) -> Result<()> { match self { Self::Rows(bounds) => bounds.validate(), + Self::Range(bounds) => bounds.validate(), } } pub fn start_is_unbounded(&self) -> bool { match self { Self::Rows(RowsFrameBounds { start, .. }) => start.is_unbounded_preceding(), + Self::Range(RangeFrameBounds { start, .. }) => start.is_unbounded_preceding(), } } pub fn end_is_unbounded(&self) -> bool { match self { Self::Rows(RowsFrameBounds { end, .. }) => end.is_unbounded_following(), + Self::Range(RangeFrameBounds { end, .. }) => end.is_unbounded_following(), } } @@ -143,16 +181,325 @@ impl FrameBounds { } } +pub trait FrameBoundsImpl { + fn validate(&self) -> Result<()>; +} + #[derive(Display, Debug, Clone, Eq, PartialEq, Hash)] #[display("ROWS BETWEEN {start} AND {end}")] pub struct RowsFrameBounds { - pub start: FrameBound, - pub end: FrameBound, + pub start: RowsFrameBound, + pub end: RowsFrameBound, } impl RowsFrameBounds { + fn from_protobuf(bounds: &PbRowsFrameBounds) -> Result { + let start = FrameBound::::from_protobuf(bounds.get_start()?)?; + let end = FrameBound::::from_protobuf(bounds.get_end()?)?; + Ok(Self { start, end }) + } + + fn to_protobuf(&self) -> PbRowsFrameBounds { + PbRowsFrameBounds { + start: Some(self.start.to_protobuf()), + end: Some(self.end.to_protobuf()), + } + } +} + +impl RowsFrameBounds { + /// Check if the `ROWS` frame is canonical. + /// + /// A canonical `ROWS` frame is defined as: + /// + /// - Its bounds are valid (see [`Self::validate`]). + /// - It contains the current row. + pub fn is_canonical(&self) -> bool { + self.validate().is_ok() && { + let start = self.start.to_offset(); + let end = self.end.to_offset(); + start.unwrap_or(0) <= 0 && end.unwrap_or(0) >= 0 + } + } + + /// Get the number of preceding rows. + pub fn n_preceding_rows(&self) -> Option { + match (&self.start, &self.end) { + (UnboundedPreceding, _) => None, + (Preceding(n1), Preceding(n2)) => Some(*n1.max(n2)), + (Preceding(n), _) => Some(*n), + (CurrentRow | Following(_) | UnboundedFollowing, _) => Some(0), + } + } + + /// Get the number of following rows. + pub fn n_following_rows(&self) -> Option { + match (&self.start, &self.end) { + (_, UnboundedFollowing) => None, + (Following(n1), Following(n2)) => Some(*n1.max(n2)), + (_, Following(n)) => Some(*n), + (_, CurrentRow | Preceding(_) | UnboundedPreceding) => Some(0), + } + } +} + +impl FrameBoundsImpl for RowsFrameBounds { fn validate(&self) -> Result<()> { - FrameBound::validate_bounds(&self.start, &self.end) + FrameBound::validate_bounds(&self.start, &self.end, |_| Ok(())) + } +} + +#[derive(Debug, Clone, Eq, PartialEq, Hash)] +pub struct RangeFrameBounds { + pub order_data_type: DataType, + pub order_type: OrderType, + pub offset_data_type: DataType, + pub start: RangeFrameBound, + pub end: RangeFrameBound, +} + +impl RangeFrameBounds { + fn from_protobuf(bounds: &PbRangeFrameBounds) -> Result { + let order_data_type = DataType::from(bounds.get_order_data_type()?); + let order_type = OrderType::from_protobuf(bounds.get_order_type()?); + let offset_data_type = DataType::from(bounds.get_offset_data_type()?); + let start = FrameBound::::from_protobuf( + bounds.get_start()?, + &order_data_type, + &offset_data_type, + )?; + let end = FrameBound::::from_protobuf( + bounds.get_end()?, + &order_data_type, + &offset_data_type, + )?; + Ok(Self { + order_data_type, + order_type, + offset_data_type, + start, + end, + }) + } + + fn to_protobuf(&self) -> PbRangeFrameBounds { + PbRangeFrameBounds { + start: Some(self.start.to_protobuf()), + end: Some(self.end.to_protobuf()), + order_data_type: Some(self.order_data_type.to_protobuf()), + order_type: Some(self.order_type.to_protobuf()), + offset_data_type: Some(self.offset_data_type.to_protobuf()), + } + } +} + +/// The wrapper type for [`ScalarImpl`] range frame offset, containing +/// two expressions to help adding and subtracting the offset. +#[derive(Debug, Clone, Educe)] +#[educe(PartialEq, Eq, Hash)] +pub struct RangeFrameOffset { + /// The original offset value. + offset: ScalarImpl, + /// Built expression for `$0 + offset`. + #[educe(PartialEq(ignore), Hash(ignore))] + add_expr: Option>, + /// Built expression for `$0 - offset`. + #[educe(PartialEq(ignore), Hash(ignore))] + sub_expr: Option>, +} + +impl RangeFrameOffset { + pub fn new(offset: ScalarImpl) -> Self { + Self { + offset, + add_expr: None, + sub_expr: None, + } + } + + fn build_exprs( + &mut self, + order_data_type: &DataType, + offset_data_type: &DataType, + ) -> Result<()> { + use risingwave_pb::expr::expr_node::PbType as PbExprType; + + let input_expr = InputRefExpression::new(order_data_type.clone(), 0); + let offset_expr = + LiteralExpression::new(offset_data_type.clone(), Some(self.offset.clone())); + self.add_expr = Some(Arc::new(build_func( + PbExprType::Add, + order_data_type.clone(), + vec![input_expr.clone().boxed(), offset_expr.clone().boxed()], + )?)); + self.sub_expr = Some(Arc::new(build_func( + PbExprType::Subtract, + order_data_type.clone(), + vec![input_expr.boxed(), offset_expr.boxed()], + )?)); + Ok(()) + } + + pub fn new_for_test( + offset: ScalarImpl, + order_data_type: &DataType, + offset_data_type: &DataType, + ) -> Self { + let mut offset = Self::new(offset); + offset + .build_exprs(order_data_type, offset_data_type) + .unwrap(); + offset + } +} + +impl Deref for RangeFrameOffset { + type Target = ScalarImpl; + + fn deref(&self) -> &Self::Target { + &self.offset + } +} + +impl Display for RangeFrameBounds { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "RANGE BETWEEN {} AND {}", + self.start.for_display(), + self.end.for_display() + )?; + Ok(()) + } +} + +impl FrameBoundsImpl for RangeFrameBounds { + fn validate(&self) -> Result<()> { + fn validate_non_negative(val: impl IsNegative + Display) -> Result<()> { + if val.is_negative() { + bail!( + "frame bound offset should be non-negative, but {} is given", + val + ); + } + Ok(()) + } + + FrameBound::validate_bounds(&self.start, &self.end, |offset| { + match offset.as_scalar_ref_impl() { + // TODO(rc): use decl macro? + ScalarRefImpl::Int16(val) => validate_non_negative(val)?, + ScalarRefImpl::Int32(val) => validate_non_negative(val)?, + ScalarRefImpl::Int64(val) => validate_non_negative(val)?, + ScalarRefImpl::Float32(val) => validate_non_negative(val)?, + ScalarRefImpl::Float64(val) => validate_non_negative(val)?, + ScalarRefImpl::Decimal(val) => validate_non_negative(val)?, + ScalarRefImpl::Interval(val) => { + if !val.is_never_negative() { + bail!( + "for frame bound offset of type `interval`, each field should be non-negative, but {} is given", + val + ); + } + if matches!(self.order_data_type, DataType::Timestamptz) { + // for `timestamptz`, we only support offset without `month` and `day` fields + if val.months() != 0 || val.days() != 0 { + bail!( + "for frame order column of type `timestamptz`, offset should not have non-zero `month` and `day`", + ); + } + } + }, + _ => unreachable!("other order column data types are not supported and should be banned in frontend"), + } + Ok(()) + }) + } +} + +impl RangeFrameBounds { + /// Get the frame start for a given order column value. + /// + /// ## Examples + /// + /// For the following frames: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + /// ORDER BY x DESC RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW + /// ``` + /// + /// For any CURRENT ROW with any order value, the frame start is always the first-most row, which is + /// represented by [`Sentinelled::Smallest`]. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW + /// ``` + /// + /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `90`. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x DESC RANGE BETWEEN 10 PRECEDING AND CURRENT ROW + /// ``` + /// + /// For CURRENT ROW with order value `100`, the frame start is the **FIRST** row with order value `110`. + pub fn frame_start_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.start.for_calc().bound_of(order_value, self.order_type) + } + + /// Get the frame end for a given order column value. It's very similar to `frame_start_of`, just with + /// everything on the other direction. + pub fn frame_end_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.end.for_calc().bound_of(order_value, self.order_type) + } + + /// Get the order value of the CURRENT ROW of the first frame that includes the given order value. + /// + /// ## Examples + /// + /// For the following frames: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + /// ``` + /// + /// For any given order value, the first CURRENT ROW is always the first-most row, which is + /// represented by [`Sentinelled::Smallest`]. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x ASC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING + /// ``` + /// + /// For a given order value `100`, the first CURRENT ROW should have order value `90`. + /// + /// For the following frame: + /// + /// ```sql + /// ORDER BY x DESC RANGE BETWEEN CURRENT ROW AND 10 FOLLOWING + /// ``` + /// + /// For a given order value `100`, the first CURRENT ROW should have order value `110`. + pub fn first_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.end + .for_calc() + .reverse() + .bound_of(order_value, self.order_type) + } + + /// Get the order value of the CURRENT ROW of the last frame that includes the given order value. + /// It's very similar to `first_curr_of`, just with everything on the other direction. + pub fn last_curr_of(&self, order_value: impl ToOwnedDatum) -> Sentinelled { + self.start + .for_calc() + .reverse() + .bound_of(order_value, self.order_type) } } @@ -168,11 +515,27 @@ pub enum FrameBound { UnboundedFollowing, } +pub type RowsFrameBound = FrameBound; +pub type RangeFrameBound = FrameBound; + impl FrameBound { - fn validate_bounds(start: &Self, end: &Self) -> Result<()> { + fn offset_value(&self) -> Option<&T> { + match self { + UnboundedPreceding | UnboundedFollowing | CurrentRow => None, + Preceding(offset) | Following(offset) => Some(offset), + } + } + + fn validate_bounds( + start: &Self, + end: &Self, + offset_checker: impl Fn(&T) -> Result<()>, + ) -> Result<()> { match (start, end) { (_, UnboundedPreceding) => bail!("frame end cannot be UNBOUNDED PRECEDING"), - (UnboundedFollowing, _) => bail!("frame start cannot be UNBOUNDED FOLLOWING"), + (UnboundedFollowing, _) => { + bail!("frame start cannot be UNBOUNDED FOLLOWING") + } (Following(_), CurrentRow) | (Following(_), Preceding(_)) => { bail!("frame starting from following row cannot have preceding rows") } @@ -181,49 +544,85 @@ impl FrameBound { } _ => {} } + + for bound in [start, end] { + if let Some(offset) = bound.offset_value() { + offset_checker(offset)?; + } + } + Ok(()) } + + pub fn map(self, f: impl Fn(T) -> U) -> FrameBound { + match self { + UnboundedPreceding => UnboundedPreceding, + Preceding(offset) => Preceding(f(offset)), + CurrentRow => CurrentRow, + Following(offset) => Following(f(offset)), + UnboundedFollowing => UnboundedFollowing, + } + } } -impl FrameBound { - pub fn from_protobuf(bound: &PbBound) -> Result { +impl FrameBound +where + T: Copy, +{ + fn reverse(self) -> FrameBound { + match self { + UnboundedPreceding => UnboundedFollowing, + Preceding(offset) => Following(offset), + CurrentRow => CurrentRow, + Following(offset) => Preceding(offset), + UnboundedFollowing => UnboundedPreceding, + } + } +} + +impl RowsFrameBound { + fn from_protobuf_legacy(bound: &PbBound) -> Result { use risingwave_pb::expr::window_frame::bound::PbOffset; - use risingwave_pb::expr::window_frame::PbBoundType; let offset = bound.get_offset()?; let bound = match offset { - PbOffset::Integer(offset) => match bound.get_type()? { - PbBoundType::Unspecified => bail!("unspecified type of `FrameBound`"), - PbBoundType::UnboundedPreceding => Self::UnboundedPreceding, - PbBoundType::Preceding => Self::Preceding(*offset as usize), - PbBoundType::CurrentRow => Self::CurrentRow, - PbBoundType::Following => Self::Following(*offset as usize), - PbBoundType::UnboundedFollowing => Self::UnboundedFollowing, - }, - PbOffset::Datum(_) => bail!("offset of `FrameBound` must be `Integer`"), + PbOffset::Integer(offset) => Self::from_protobuf(&PbRowsFrameBound { + r#type: bound.get_type()? as _, + offset: Some(*offset), + })?, + PbOffset::Datum(_) => bail!("offset of `RowsFrameBound` must be `Integer`"), }; Ok(bound) } - pub fn to_protobuf(&self) -> PbBound { - use risingwave_pb::expr::window_frame::bound::PbOffset; - use risingwave_pb::expr::window_frame::PbBoundType; + fn from_protobuf(bound: &PbRowsFrameBound) -> Result { + let bound = match bound.get_type()? { + PbBoundType::Unspecified => bail!("unspecified type of `RowsFrameBound`"), + PbBoundType::UnboundedPreceding => Self::UnboundedPreceding, + PbBoundType::Preceding => Self::Preceding(*bound.get_offset()? as usize), + PbBoundType::CurrentRow => Self::CurrentRow, + PbBoundType::Following => Self::Following(*bound.get_offset()? as usize), + PbBoundType::UnboundedFollowing => Self::UnboundedFollowing, + }; + Ok(bound) + } + fn to_protobuf(&self) -> PbRowsFrameBound { let (r#type, offset) = match self { - Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, PbOffset::Integer(0)), - Self::Preceding(offset) => (PbBoundType::Preceding, PbOffset::Integer(*offset as _)), - Self::CurrentRow => (PbBoundType::CurrentRow, PbOffset::Integer(0)), - Self::Following(offset) => (PbBoundType::Following, PbOffset::Integer(*offset as _)), - Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, PbOffset::Integer(0)), + Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None), + Self::Preceding(offset) => (PbBoundType::Preceding, Some(*offset as _)), + Self::CurrentRow => (PbBoundType::CurrentRow, None), + Self::Following(offset) => (PbBoundType::Following, Some(*offset as _)), + Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None), }; - PbBound { + PbRowsFrameBound { r#type: r#type as _, - offset: Some(offset), + offset, } } } -impl FrameBound { +impl RowsFrameBound { /// Convert the bound to sized offset from current row. `None` if the bound is unbounded. pub fn to_offset(&self) -> Option { match self { @@ -233,15 +632,117 @@ impl FrameBound { Following(n) => Some(*n as isize), } } +} - /// View the bound as frame start, and get the number of preceding rows. - pub fn n_preceding_rows(&self) -> Option { - self.to_offset().map(|x| x.min(0).unsigned_abs()) +impl RangeFrameBound { + fn from_protobuf( + bound: &PbRangeFrameBound, + order_data_type: &DataType, + offset_data_type: &DataType, + ) -> Result { + let bound = match bound.get_type()? { + PbBoundType::Unspecified => bail!("unspecified type of `RangeFrameBound`"), + PbBoundType::UnboundedPreceding => Self::UnboundedPreceding, + PbBoundType::CurrentRow => Self::CurrentRow, + PbBoundType::UnboundedFollowing => Self::UnboundedFollowing, + bound_type @ (PbBoundType::Preceding | PbBoundType::Following) => { + let offset_value = Datum::from_protobuf(bound.get_offset()?, offset_data_type) + .context("offset `Datum` is not decodable")? + .context("offset of `RangeFrameBound` must be non-NULL")?; + let mut offset = RangeFrameOffset::new(offset_value); + offset.build_exprs(order_data_type, offset_data_type)?; + if bound_type == PbBoundType::Preceding { + Self::Preceding(offset) + } else { + Self::Following(offset) + } + } + }; + Ok(bound) } - /// View the bound as frame end, and get the number of following rows. - pub fn n_following_rows(&self) -> Option { - self.to_offset().map(|x| x.max(0) as usize) + fn to_protobuf(&self) -> PbRangeFrameBound { + let (r#type, offset) = match self { + Self::UnboundedPreceding => (PbBoundType::UnboundedPreceding, None), + Self::Preceding(offset) => ( + PbBoundType::Preceding, + Some(Some(offset.as_scalar_ref_impl()).to_protobuf()), + ), + Self::CurrentRow => (PbBoundType::CurrentRow, None), + Self::Following(offset) => ( + PbBoundType::Following, + Some(Some(offset.as_scalar_ref_impl()).to_protobuf()), + ), + Self::UnboundedFollowing => (PbBoundType::UnboundedFollowing, None), + }; + PbRangeFrameBound { + r#type: r#type as _, + offset, + } + } +} + +impl RangeFrameBound { + fn for_display(&self) -> FrameBound { + match self { + UnboundedPreceding => UnboundedPreceding, + Preceding(offset) => Preceding(offset.as_scalar_ref_impl().to_text()), + CurrentRow => CurrentRow, + Following(offset) => Following(offset.as_scalar_ref_impl().to_text()), + UnboundedFollowing => UnboundedFollowing, + } + } + + fn for_calc(&self) -> FrameBound> { + match self { + UnboundedPreceding => UnboundedPreceding, + Preceding(offset) => Preceding(RangeFrameOffsetRef { + add_expr: offset.add_expr.as_ref().unwrap().as_ref(), + sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(), + }), + CurrentRow => CurrentRow, + Following(offset) => Following(RangeFrameOffsetRef { + add_expr: offset.add_expr.as_ref().unwrap().as_ref(), + sub_expr: offset.sub_expr.as_ref().unwrap().as_ref(), + }), + UnboundedFollowing => UnboundedFollowing, + } + } +} + +#[derive(Debug, Educe)] +#[educe(Clone, Copy)] +pub struct RangeFrameOffsetRef<'a> { + /// Built expression for `$0 + offset`. + add_expr: &'a dyn Expression, + /// Built expression for `$0 - offset`. + sub_expr: &'a dyn Expression, +} + +impl FrameBound> { + fn bound_of(self, order_value: impl ToOwnedDatum, order_type: OrderType) -> Sentinelled { + let expr = match (self, order_type.direction()) { + (UnboundedPreceding, _) => return Sentinelled::Smallest, + (UnboundedFollowing, _) => return Sentinelled::Largest, + (CurrentRow, _) => return Sentinelled::Normal(order_value.to_owned_datum()), + (Preceding(offset), Direction::Ascending) + | (Following(offset), Direction::Descending) => { + // should SUBTRACT the offset + offset.sub_expr + } + (Following(offset), Direction::Ascending) + | (Preceding(offset), Direction::Descending) => { + // should ADD the offset + offset.add_expr + } + }; + let row = OwnedRow::new(vec![order_value.to_owned_datum()]); + Sentinelled::Normal( + expr.eval_row(&row) + .now_or_never() + .expect("frame bound calculation should finish immediately") + .expect("just simple calculation, should succeed"), // TODO(rc): handle overflow + ) } } @@ -256,7 +757,7 @@ pub enum FrameExclusion { } impl FrameExclusion { - pub fn from_protobuf(exclusion: PbExclusion) -> Result { + fn from_protobuf(exclusion: PbExclusion) -> Result { let excl = match exclusion { PbExclusion::Unspecified => bail!("unspecified type of `FrameExclusion`"), PbExclusion::CurrentRow => Self::CurrentRow, @@ -265,10 +766,99 @@ impl FrameExclusion { Ok(excl) } - pub fn to_protobuf(self) -> PbExclusion { + fn to_protobuf(self) -> PbExclusion { match self { Self::CurrentRow => PbExclusion::CurrentRow, Self::NoOthers => PbExclusion::NoOthers, } } } + +#[cfg(test)] +mod tests { + + use super::*; + + #[test] + fn test_rows_frame_bounds() { + let bounds = RowsFrameBounds { + start: Preceding(1), + end: CurrentRow, + }; + assert!(bounds.validate().is_ok()); + assert!(bounds.is_canonical()); + assert_eq!(bounds.start.to_offset(), Some(-1)); + assert_eq!(bounds.end.to_offset(), Some(0)); + assert_eq!(bounds.n_preceding_rows(), Some(1)); + assert_eq!(bounds.n_following_rows(), Some(0)); + + let bounds = RowsFrameBounds { + start: CurrentRow, + end: Following(1), + }; + assert!(bounds.validate().is_ok()); + assert!(bounds.is_canonical()); + assert_eq!(bounds.start.to_offset(), Some(0)); + assert_eq!(bounds.end.to_offset(), Some(1)); + assert_eq!(bounds.n_preceding_rows(), Some(0)); + assert_eq!(bounds.n_following_rows(), Some(1)); + + let bounds = RowsFrameBounds { + start: UnboundedPreceding, + end: Following(10), + }; + assert!(bounds.validate().is_ok()); + assert!(bounds.is_canonical()); + assert_eq!(bounds.start.to_offset(), None); + assert_eq!(bounds.end.to_offset(), Some(10)); + assert_eq!(bounds.n_preceding_rows(), None); + assert_eq!(bounds.n_following_rows(), Some(10)); + + let bounds = RowsFrameBounds { + start: Preceding(10), + end: UnboundedFollowing, + }; + assert!(bounds.validate().is_ok()); + assert!(bounds.is_canonical()); + assert_eq!(bounds.start.to_offset(), Some(-10)); + assert_eq!(bounds.end.to_offset(), None); + assert_eq!(bounds.n_preceding_rows(), Some(10)); + assert_eq!(bounds.n_following_rows(), None); + + let bounds = RowsFrameBounds { + start: Preceding(1), + end: Preceding(10), + }; + assert!(bounds.validate().is_ok()); + assert!(!bounds.is_canonical()); + assert_eq!(bounds.start.to_offset(), Some(-1)); + assert_eq!(bounds.end.to_offset(), Some(-10)); + assert_eq!(bounds.n_preceding_rows(), Some(10)); + assert_eq!(bounds.n_following_rows(), Some(0)); + + let bounds = RowsFrameBounds { + start: Following(10), + end: Following(1), + }; + assert!(bounds.validate().is_ok()); + assert!(!bounds.is_canonical()); + assert_eq!(bounds.start.to_offset(), Some(10)); + assert_eq!(bounds.end.to_offset(), Some(1)); + assert_eq!(bounds.n_preceding_rows(), Some(0)); + assert_eq!(bounds.n_following_rows(), Some(10)); + + let bounds = RowsFrameBounds { + start: UnboundedFollowing, + end: Following(10), + }; + assert!(bounds.validate().is_err()); + assert!(!bounds.is_canonical()); + + let bounds = RowsFrameBounds { + start: Preceding(10), + end: UnboundedPreceding, + }; + assert!(bounds.validate().is_err()); + assert!(!bounds.is_canonical()); + } +} diff --git a/src/expr/core/src/window_function/state/aggregate.rs b/src/expr/core/src/window_function/state/aggregate.rs index 09555eecf9201..08689bff6da72 100644 --- a/src/expr/core/src/window_function/state/aggregate.rs +++ b/src/expr/core/src/window_function/state/aggregate.rs @@ -22,66 +22,93 @@ use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::{bail, must_match}; use smallvec::SmallVec; -use super::buffer::WindowBuffer; -use super::{StateEvictHint, StateKey, StatePos, WindowState}; +use super::buffer::{RangeWindow, RowsWindow, WindowBuffer, WindowImpl}; +use super::{BoxedWindowState, StateEvictHint, StateKey, StatePos, WindowState}; use crate::aggregate::{ AggArgs, AggCall, AggregateFunction, AggregateState as AggImplState, BoxedAggregateFunction, }; use crate::sig::FUNCTION_REGISTRY; -use crate::window_function::{WindowFuncCall, WindowFuncKind}; +use crate::window_function::{FrameBounds, WindowFuncCall, WindowFuncKind}; use crate::Result; -pub struct AggregateState { +type StateValue = SmallVec<[Datum; 2]>; + +struct AggregateState +where + W: WindowImpl, +{ agg_func: BoxedAggregateFunction, agg_impl: AggImpl, arg_data_types: Vec, - buffer: WindowBuffer>, + buffer: WindowBuffer, buffer_heap_size: KvSize, } -impl AggregateState { - pub fn new(call: &WindowFuncCall) -> Result { - if call.frame.bounds.validate().is_err() { - bail!("the window frame must be valid"); - } - let agg_kind = must_match!(call.kind, WindowFuncKind::Aggregate(agg_kind) => agg_kind); - let arg_data_types = call.args.arg_types().to_vec(); - let agg_call = AggCall { - kind: agg_kind, - args: match &call.args { - // convert args to [0] or [0, 1] - AggArgs::None => AggArgs::None, - AggArgs::Unary(data_type, _) => AggArgs::Unary(data_type.to_owned(), 0), - AggArgs::Binary(data_types, _) => AggArgs::Binary(data_types.to_owned(), [0, 1]), - }, - return_type: call.return_type.clone(), - column_orders: Vec::new(), // the input is already sorted - // TODO(rc): support filter on window function call - filter: None, - // TODO(rc): support distinct on window function call? PG doesn't support it either. - distinct: false, - direct_args: vec![], +pub(super) fn new(call: &WindowFuncCall) -> Result { + if call.frame.bounds.validate().is_err() { + bail!("the window frame must be valid"); + } + let agg_kind = must_match!(call.kind, WindowFuncKind::Aggregate(agg_kind) => agg_kind); + let arg_data_types = call.args.arg_types().to_vec(); + let agg_call = AggCall { + kind: agg_kind, + args: match &call.args { + // convert args to [0] or [0, 1] + AggArgs::None => AggArgs::None, + AggArgs::Unary(data_type, _) => AggArgs::Unary(data_type.to_owned(), 0), + AggArgs::Binary(data_types, _) => AggArgs::Binary(data_types.to_owned(), [0, 1]), + }, + return_type: call.return_type.clone(), + column_orders: Vec::new(), // the input is already sorted + // TODO(rc): support filter on window function call + filter: None, + // TODO(rc): support distinct on window function call? PG doesn't support it either. + distinct: false, + direct_args: vec![], + }; + let agg_func_sig = FUNCTION_REGISTRY + .get(agg_kind, &arg_data_types, &call.return_type) + .expect("the agg func must exist"); + let agg_func = agg_func_sig.build_aggregate(&agg_call)?; + let (agg_impl, enable_delta) = + if agg_func_sig.is_retractable() && call.frame.exclusion.is_no_others() { + let init_state = agg_func.create_state(); + (AggImpl::Incremental(init_state), true) + } else { + (AggImpl::Full, false) }; - let agg_func_sig = FUNCTION_REGISTRY - .get(agg_kind, &arg_data_types, &call.return_type) - .expect("the agg func must exist"); - let agg_func = agg_func_sig.build_aggregate(&agg_call)?; - let (agg_impl, enable_delta) = - if agg_func_sig.is_retractable() && call.frame.exclusion.is_no_others() { - let init_state = agg_func.create_state(); - (AggImpl::Incremental(init_state), true) - } else { - (AggImpl::Full, false) - }; - Ok(Self { + + let this = match &call.frame.bounds { + FrameBounds::Rows(frame_bounds) => Box::new(AggregateState { agg_func, agg_impl, arg_data_types, - buffer: WindowBuffer::new(call.frame.clone(), enable_delta), + buffer: WindowBuffer::>::new( + RowsWindow::new(frame_bounds.clone()), + call.frame.exclusion, + enable_delta, + ), buffer_heap_size: KvSize::new(), - }) - } + }) as BoxedWindowState, + FrameBounds::Range(frame_bounds) => Box::new(AggregateState { + agg_func, + agg_impl, + arg_data_types, + buffer: WindowBuffer::>::new( + RangeWindow::new(frame_bounds.clone()), + call.frame.exclusion, + enable_delta, + ), + buffer_heap_size: KvSize::new(), + }) as BoxedWindowState, + }; + Ok(this) +} +impl AggregateState +where + W: WindowImpl, +{ fn slide_inner(&mut self) -> StateEvictHint { let removed_keys: BTreeSet<_> = self .buffer @@ -107,7 +134,10 @@ impl AggregateState { } } -impl WindowState for AggregateState { +impl WindowState for AggregateState +where + W: WindowImpl, +{ fn append(&mut self, key: StateKey, args: SmallVec<[Datum; 2]>) { args.iter().for_each(|arg| { self.buffer_heap_size.add_val(arg); @@ -156,7 +186,10 @@ impl WindowState for AggregateState { } } -impl EstimateSize for AggregateState { +impl EstimateSize for AggregateState +where + W: WindowImpl, +{ fn estimated_heap_size(&self) -> usize { // estimate `VecDeque` of `StreamWindowBuffer` internal size // https://github.com/risingwavelabs/risingwave/issues/9713 diff --git a/src/expr/core/src/window_function/state/buffer.rs b/src/expr/core/src/window_function/state/buffer.rs index 3edb6d7adc164..ba064042c0021 100644 --- a/src/expr/core/src/window_function/state/buffer.rs +++ b/src/expr/core/src/window_function/state/buffer.rs @@ -15,43 +15,55 @@ use std::collections::VecDeque; use std::ops::Range; +use educe::Educe; use risingwave_common::array::Op; +use risingwave_common::types::Sentinelled; +use risingwave_common::util::memcmp_encoding; use super::range_utils::range_except; +use super::StateKey; use crate::window_function::state::range_utils::range_diff; -use crate::window_function::{Frame, FrameBounds, FrameExclusion}; - -struct Entry { - key: K, - value: V, -} +use crate::window_function::{FrameExclusion, RangeFrameBounds, RowsFrameBounds}; /// A common sliding window buffer. -pub struct WindowBuffer { - frame: Frame, - buffer: VecDeque>, +pub(super) struct WindowBuffer { + window_impl: W, + frame_exclusion: FrameExclusion, + buffer: VecDeque>, curr_idx: usize, left_idx: usize, // inclusive, note this can be > `curr_idx` right_excl_idx: usize, // exclusive, note this can be <= `curr_idx` - curr_delta: Option>, + curr_delta: Option>, +} + +/// A key-value pair in the buffer. +struct Entry { + key: K, + value: V, } /// Note: A window frame can be pure preceding, pure following, or acrossing the _current row_. -pub struct CurrWindow<'a, K> { +pub(super) struct CurrWindow<'a, K> { pub key: Option<&'a K>, + + // XXX(rc): Maybe will be used in the future, let's keep it for now. + #[cfg_attr(not(test), expect(dead_code))] + /// The preceding half of the current window is saturated. pub preceding_saturated: bool, + /// The following half of the current window is saturated. pub following_saturated: bool, } -impl WindowBuffer { - pub fn new(frame: Frame, enable_delta: bool) -> Self { - assert!(frame.bounds.validate().is_ok()); +impl WindowBuffer { + pub fn new(window_impl: W, frame_exclusion: FrameExclusion, enable_delta: bool) -> Self { if enable_delta { // TODO(rc): currently only support `FrameExclusion::NoOthers` for delta - assert!(frame.exclusion.is_no_others()); + assert!(frame_exclusion.is_no_others()); } + Self { - frame, + window_impl, + frame_exclusion, buffer: Default::default(), curr_idx: 0, left_idx: 0, @@ -64,66 +76,29 @@ impl WindowBuffer { } } - fn preceding_saturated(&self) -> bool { - self.curr_key().is_some() - && match &self.frame.bounds { - FrameBounds::Rows(bounds) => { - let start_off = bounds.start.to_offset(); - if let Some(start_off) = start_off { - if start_off >= 0 { - true // pure following frame, always preceding-saturated - } else { - // FIXME(rc): Clippy rule `clippy::nonminimal_bool` is misreporting that - // the following can be simplified. - #[allow(clippy::nonminimal_bool)] - { - assert!(self.curr_idx >= self.left_idx); - } - self.curr_idx - self.left_idx >= start_off.unsigned_abs() - } - } else { - false // unbounded frame start, never preceding-saturated - } - } - } - } - - fn following_saturated(&self) -> bool { - self.curr_key().is_some() - && match &self.frame.bounds { - FrameBounds::Rows(bounds) => { - let end_off = bounds.end.to_offset(); - if let Some(end_off) = end_off { - if end_off <= 0 { - true // pure preceding frame, always following-saturated - } else { - // FIXME(rc): Ditto. - #[allow(clippy::nonminimal_bool)] - { - assert!(self.right_excl_idx > 0); - assert!(self.right_excl_idx > self.curr_idx); - assert!(self.right_excl_idx <= self.buffer.len()); - } - self.right_excl_idx - 1 - self.curr_idx >= end_off as usize - } - } else { - false // unbounded frame end, never following-saturated - } - } - } + /// Get the smallest key that is still kept in the buffer. + /// Returns `None` if there's nothing yet. + pub fn smallest_key(&self) -> Option<&W::Key> { + self.buffer.front().map(|Entry { key, .. }| key) } /// Get the key part of the current row. - pub fn curr_key(&self) -> Option<&K> { + pub fn curr_key(&self) -> Option<&W::Key> { self.buffer.get(self.curr_idx).map(|Entry { key, .. }| key) } /// Get the current window info. - pub fn curr_window(&self) -> CurrWindow<'_, K> { + pub fn curr_window(&self) -> CurrWindow<'_, W::Key> { + let buffer_ref = BufferRef { + buffer: &self.buffer, + curr_idx: self.curr_idx, + left_idx: self.left_idx, + right_excl_idx: self.right_excl_idx, + }; CurrWindow { key: self.curr_key(), - preceding_saturated: self.preceding_saturated(), - following_saturated: self.following_saturated(), + preceding_saturated: self.window_impl.preceding_saturated(buffer_ref), + following_saturated: self.window_impl.following_saturated(buffer_ref), } } @@ -133,7 +108,7 @@ impl WindowBuffer { fn curr_window_exclusion(&self) -> Range { // TODO(rc): should intersect with `curr_window_outer` to be more accurate - match self.frame.exclusion { + match self.frame_exclusion { FrameExclusion::CurrentRow => self.curr_idx..self.curr_idx + 1, FrameExclusion::NoOthers => self.curr_idx..self.curr_idx, } @@ -146,7 +121,7 @@ impl WindowBuffer { } /// Iterate over values in the current window. - pub fn curr_window_values(&self) -> impl Iterator { + pub fn curr_window_values(&self) -> impl Iterator { assert!(self.left_idx <= self.right_excl_idx); assert!(self.right_excl_idx <= self.buffer.len()); @@ -159,69 +134,17 @@ impl WindowBuffer { /// Consume the delta of values comparing the current window to the previous window. /// The delta is not guaranteed to be sorted, especially when frame exclusion is not `NoOthers`. - pub fn consume_curr_window_values_delta(&mut self) -> impl Iterator + '_ { + pub fn consume_curr_window_values_delta( + &mut self, + ) -> impl Iterator + '_ { self.curr_delta .as_mut() .expect("delta mode should be enabled") .drain(..) } - fn recalculate_left_right(&mut self) { - // TODO(rc): For the sake of simplicity, we just recalculate the left and right indices from - // `curr_idx`, rather than trying to update them incrementally. The complexity is O(n) for - // `Frame::Range` where n is the length of the buffer, for now it doesn't matter. - - if self.buffer.is_empty() { - self.left_idx = 0; - self.right_excl_idx = 0; - } - - match &self.frame.bounds { - FrameBounds::Rows(bounds) => { - let start_off = bounds.start.to_offset(); - let end_off = bounds.end.to_offset(); - if let Some(start_off) = start_off { - let logical_left_idx = self.curr_idx as isize + start_off; - if logical_left_idx >= 0 { - self.left_idx = std::cmp::min(logical_left_idx as usize, self.buffer.len()); - } else { - self.left_idx = 0; - } - } else { - // unbounded start - self.left_idx = 0; - } - if let Some(end_off) = end_off { - let logical_right_excl_idx = self.curr_idx as isize + end_off + 1; - if logical_right_excl_idx >= 0 { - self.right_excl_idx = - std::cmp::min(logical_right_excl_idx as usize, self.buffer.len()); - } else { - self.right_excl_idx = 0; - } - } else { - // unbounded end - self.right_excl_idx = self.buffer.len(); - } - } - } - } - - fn maintain_delta(&mut self, old_outer: Range, new_outer: Range) { - debug_assert!(self.frame.exclusion.is_no_others()); - - let (outer_removed, outer_added) = range_diff(old_outer.clone(), new_outer.clone()); - let delta = self.curr_delta.as_mut().unwrap(); - for idx in outer_removed.iter().cloned().flatten() { - delta.push((Op::Delete, self.buffer[idx].value.clone())); - } - for idx in outer_added.iter().cloned().flatten() { - delta.push((Op::Insert, self.buffer[idx].value.clone())); - } - } - /// Append a key-value pair to the buffer. - pub fn append(&mut self, key: K, value: V) { + pub fn append(&mut self, key: W::Key, value: W::Value) { let old_outer = self.curr_window_outer(); self.buffer.push_back(Entry { key, value }); @@ -232,15 +155,9 @@ impl WindowBuffer { } } - /// Get the smallest key that is still kept in the buffer. - /// Returns `None` if there's nothing yet. - pub fn smallest_key(&self) -> Option<&K> { - self.buffer.front().map(|Entry { key, .. }| key) - } - /// Slide the current window forward. /// Returns the keys that are removed from the buffer. - pub fn slide(&mut self) -> impl Iterator + '_ { + pub fn slide(&mut self) -> impl Iterator + '_ { let old_outer = self.curr_window_outer(); self.curr_idx += 1; @@ -258,6 +175,265 @@ impl WindowBuffer { .drain(0..min_needed_idx) .map(|Entry { key, value }| (key, value)) } + + fn maintain_delta(&mut self, old_outer: Range, new_outer: Range) { + debug_assert!(self.frame_exclusion.is_no_others()); + + let (outer_removed, outer_added) = range_diff(old_outer.clone(), new_outer.clone()); + let delta = self.curr_delta.as_mut().unwrap(); + for idx in outer_removed.iter().cloned().flatten() { + delta.push((Op::Delete, self.buffer[idx].value.clone())); + } + for idx in outer_added.iter().cloned().flatten() { + delta.push((Op::Insert, self.buffer[idx].value.clone())); + } + } + + fn recalculate_left_right(&mut self) { + let buffer_ref = BufferRefMut { + buffer: &self.buffer, + curr_idx: &mut self.curr_idx, + left_idx: &mut self.left_idx, + right_excl_idx: &mut self.right_excl_idx, + }; + self.window_impl.recalculate_left_right(buffer_ref); + } +} + +/// Wraps a reference to the buffer and some indices, to be used by [`WindowImpl`]s. +#[derive(Educe)] +#[educe(Clone, Copy)] +pub(super) struct BufferRef<'a, K: Ord, V: Clone> { + buffer: &'a VecDeque>, + curr_idx: usize, + left_idx: usize, + right_excl_idx: usize, +} + +/// Wraps a reference to the buffer and some mutable indices, to be used by [`WindowImpl`]s. +pub(super) struct BufferRefMut<'a, K: Ord, V: Clone> { + buffer: &'a VecDeque>, + curr_idx: &'a mut usize, + left_idx: &'a mut usize, + right_excl_idx: &'a mut usize, +} + +/// A trait for sliding window implementations. This trait is used by [`WindowBuffer`] to +/// determine the status of current window and how to slide the window. +pub(super) trait WindowImpl { + type Key: Ord; + type Value: Clone; + + /// Whether the preceding half of the current window is saturated. + /// By "saturated" we mean that every row that is possible to be in the preceding half of the + /// current window is already in the buffer. + fn preceding_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool; + + /// Whether the following half of the current window is saturated. + fn following_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool; + + /// Recalculate the left and right indices of the current window, according to the latest + /// `curr_idx`. The indices are indices in the buffer vector. + fn recalculate_left_right(&self, buffer_ref: BufferRefMut<'_, Self::Key, Self::Value>); +} + +/// The sliding window implementation for `ROWS` frames. +pub(super) struct RowsWindow { + frame_bounds: RowsFrameBounds, + _phantom: std::marker::PhantomData, + _phantom2: std::marker::PhantomData, +} + +impl RowsWindow { + pub fn new(frame_bounds: RowsFrameBounds) -> Self { + Self { + frame_bounds, + _phantom: std::marker::PhantomData, + _phantom2: std::marker::PhantomData, + } + } +} + +impl WindowImpl for RowsWindow { + type Key = K; + type Value = V; + + fn preceding_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool { + buffer_ref.curr_idx < buffer_ref.buffer.len() && { + let start_off = self.frame_bounds.start.to_offset(); + if let Some(start_off) = start_off { + if start_off >= 0 { + true // pure following frame, always preceding-saturated + } else { + // FIXME(rc): Clippy rule `clippy::nonminimal_bool` is misreporting that + // the following can be simplified. + #[allow(clippy::nonminimal_bool)] + { + assert!(buffer_ref.curr_idx >= buffer_ref.left_idx); + } + buffer_ref.curr_idx - buffer_ref.left_idx >= start_off.unsigned_abs() + } + } else { + false // unbounded frame start, never preceding-saturated + } + } + } + + fn following_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool { + buffer_ref.curr_idx < buffer_ref.buffer.len() && { + let end_off = self.frame_bounds.end.to_offset(); + if let Some(end_off) = end_off { + if end_off <= 0 { + true // pure preceding frame, always following-saturated + } else { + // FIXME(rc): Ditto. + #[allow(clippy::nonminimal_bool)] + { + assert!(buffer_ref.right_excl_idx > 0); + assert!(buffer_ref.right_excl_idx > buffer_ref.curr_idx); + assert!(buffer_ref.right_excl_idx <= buffer_ref.buffer.len()); + } + buffer_ref.right_excl_idx - 1 - buffer_ref.curr_idx >= end_off as usize + } + } else { + false // unbounded frame end, never following-saturated + } + } + } + + fn recalculate_left_right(&self, buffer_ref: BufferRefMut<'_, Self::Key, Self::Value>) { + if buffer_ref.buffer.is_empty() { + *buffer_ref.left_idx = 0; + *buffer_ref.right_excl_idx = 0; + } + + let start_off = self.frame_bounds.start.to_offset(); + let end_off = self.frame_bounds.end.to_offset(); + if let Some(start_off) = start_off { + let logical_left_idx = *buffer_ref.curr_idx as isize + start_off; + if logical_left_idx >= 0 { + *buffer_ref.left_idx = + std::cmp::min(logical_left_idx as usize, buffer_ref.buffer.len()); + } else { + *buffer_ref.left_idx = 0; + } + } else { + // unbounded start + *buffer_ref.left_idx = 0; + } + if let Some(end_off) = end_off { + let logical_right_excl_idx = *buffer_ref.curr_idx as isize + end_off + 1; + if logical_right_excl_idx >= 0 { + *buffer_ref.right_excl_idx = + std::cmp::min(logical_right_excl_idx as usize, buffer_ref.buffer.len()); + } else { + *buffer_ref.right_excl_idx = 0; + } + } else { + // unbounded end + *buffer_ref.right_excl_idx = buffer_ref.buffer.len(); + } + } +} + +/// The sliding window implementation for `RANGE` frames. +pub(super) struct RangeWindow { + frame_bounds: RangeFrameBounds, + _phantom: std::marker::PhantomData, +} + +impl RangeWindow { + pub fn new(frame_bounds: RangeFrameBounds) -> Self { + Self { + frame_bounds, + _phantom: std::marker::PhantomData, + } + } +} + +impl WindowImpl for RangeWindow { + type Key = StateKey; + type Value = V; + + fn preceding_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool { + buffer_ref.curr_idx < buffer_ref.buffer.len() && { + // XXX(rc): It seems that preceding saturation is not important, may remove later. + true + } + } + + fn following_saturated(&self, buffer_ref: BufferRef<'_, Self::Key, Self::Value>) -> bool { + buffer_ref.curr_idx < buffer_ref.buffer.len() + && { + // Left OK? (note that `left_idx` can be greater than `right_idx`) + // The following line checks whether the left value is the last one in the buffer. + // Here we adopt a conservative approach, which means we assume the next future value + // is likely to be the same as the last value in the current window, in which case + // we can't say the current window is saturated. + buffer_ref.left_idx < buffer_ref.buffer.len() /* non-zero */ - 1 + } + && { + // Right OK? Ditto. + buffer_ref.right_excl_idx < buffer_ref.buffer.len() + } + } + + fn recalculate_left_right(&self, buffer_ref: BufferRefMut<'_, Self::Key, Self::Value>) { + if buffer_ref.buffer.is_empty() { + *buffer_ref.left_idx = 0; + *buffer_ref.right_excl_idx = 0; + } + + let Some(entry) = buffer_ref.buffer.get(*buffer_ref.curr_idx) else { + // If the current index has been moved to a future position, we can't touch anything + // because the next coming key may equal to the previous one which means the left and + // right indices will be the same. + return; + }; + let curr_key = &entry.key; + + let curr_order_value = memcmp_encoding::decode_value( + &self.frame_bounds.order_data_type, + &curr_key.order_key, + self.frame_bounds.order_type, + ) + .expect("no reason to fail here because we just encoded it in memory"); + + match self.frame_bounds.frame_start_of(&curr_order_value) { + Sentinelled::Smallest => { + // unbounded frame start + assert_eq!( + *buffer_ref.left_idx, 0, + "for unbounded start, left index should always be 0" + ); + } + Sentinelled::Normal(value) => { + // bounded, find the start position + let value_enc = memcmp_encoding::encode_value(value, self.frame_bounds.order_type) + .expect("no reason to fail here"); + *buffer_ref.left_idx = buffer_ref + .buffer + .partition_point(|elem| elem.key.order_key < value_enc); + } + Sentinelled::Largest => unreachable!("frame start never be UNBOUNDED FOLLOWING"), + } + + match self.frame_bounds.frame_end_of(curr_order_value) { + Sentinelled::Largest => { + // unbounded frame end + *buffer_ref.right_excl_idx = buffer_ref.buffer.len(); + } + Sentinelled::Normal(value) => { + // bounded, find the end position + let value_enc = memcmp_encoding::encode_value(value, self.frame_bounds.order_type) + .expect("no reason to fail here"); + *buffer_ref.right_excl_idx = buffer_ref + .buffer + .partition_point(|elem| elem.key.order_key <= value_enc); + } + Sentinelled::Smallest => unreachable!("frame end never be UNBOUNDED PRECEDING"), + } + } } #[cfg(test)] @@ -265,12 +441,18 @@ mod tests { use itertools::Itertools; use super::*; - use crate::window_function::{Frame, FrameBound}; + use crate::window_function::FrameBound::{ + CurrentRow, Following, Preceding, UnboundedFollowing, UnboundedPreceding, + }; #[test] fn test_rows_frame_unbounded_preceding_to_current_row() { - let mut buffer = WindowBuffer::new( - Frame::rows(FrameBound::UnboundedPreceding, FrameBound::CurrentRow), + let mut buffer = WindowBuffer::>::new( + RowsWindow::new(RowsFrameBounds { + start: UnboundedPreceding, + end: CurrentRow, + }), + FrameExclusion::NoOthers, true, ); @@ -303,8 +485,12 @@ mod tests { #[test] fn test_rows_frame_preceding_to_current_row() { - let mut buffer = WindowBuffer::new( - Frame::rows(FrameBound::Preceding(1), FrameBound::CurrentRow), + let mut buffer = WindowBuffer::>::new( + RowsWindow::new(RowsFrameBounds { + start: Preceding(1), + end: CurrentRow, + }), + FrameExclusion::NoOthers, true, ); @@ -342,8 +528,12 @@ mod tests { #[test] fn test_rows_frame_preceding_to_preceding() { - let mut buffer = WindowBuffer::new( - Frame::rows(FrameBound::Preceding(2), FrameBound::Preceding(1)), + let mut buffer = WindowBuffer::>::new( + RowsWindow::new(RowsFrameBounds { + start: Preceding(2), + end: Preceding(1), + }), + FrameExclusion::NoOthers, true, ); @@ -384,8 +574,12 @@ mod tests { #[test] fn test_rows_frame_current_row_to_unbounded_following() { - let mut buffer = WindowBuffer::new( - Frame::rows(FrameBound::CurrentRow, FrameBound::UnboundedFollowing), + let mut buffer = WindowBuffer::>::new( + RowsWindow::new(RowsFrameBounds { + start: CurrentRow, + end: UnboundedFollowing, + }), + FrameExclusion::NoOthers, true, ); @@ -422,8 +616,12 @@ mod tests { #[test] fn test_rows_frame_current_row_to_following() { - let mut buffer = WindowBuffer::new( - Frame::rows(FrameBound::CurrentRow, FrameBound::Following(1)), + let mut buffer = WindowBuffer::>::new( + RowsWindow::new(RowsFrameBounds { + start: CurrentRow, + end: Following(1), + }), + FrameExclusion::NoOthers, true, ); @@ -468,8 +666,12 @@ mod tests { #[test] fn test_rows_frame_following_to_following() { - let mut buffer = WindowBuffer::new( - Frame::rows(FrameBound::Following(1), FrameBound::Following(2)), + let mut buffer = WindowBuffer::>::new( + RowsWindow::new(RowsFrameBounds { + start: Following(1), + end: Following(2), + }), + FrameExclusion::NoOthers, true, ); @@ -511,12 +713,12 @@ mod tests { #[test] fn test_rows_frame_exclude_current_row() { - let mut buffer = WindowBuffer::new( - Frame::rows_with_exclusion( - FrameBound::UnboundedPreceding, - FrameBound::CurrentRow, - FrameExclusion::CurrentRow, - ), + let mut buffer = WindowBuffer::>::new( + RowsWindow::new(RowsFrameBounds { + start: UnboundedPreceding, + end: CurrentRow, + }), + FrameExclusion::CurrentRow, false, ); diff --git a/src/expr/core/src/window_function/state/mod.rs b/src/expr/core/src/window_function/state/mod.rs index 37ee086ca7ba4..69162f33e5f5b 100644 --- a/src/expr/core/src/window_function/state/mod.rs +++ b/src/expr/core/src/window_function/state/mod.rs @@ -17,7 +17,7 @@ use std::collections::BTreeSet; use itertools::Itertools; use risingwave_common::estimate_size::EstimateSize; use risingwave_common::row::OwnedRow; -use risingwave_common::types::{Datum, DefaultOrdered, Sentinelled}; +use risingwave_common::types::{Datum, DefaultOrdered}; use risingwave_common::util::memcmp_encoding::MemcmpEncoded; use smallvec::SmallVec; @@ -36,12 +36,6 @@ pub struct StateKey { pub pk: DefaultOrdered, } -impl From for Sentinelled { - fn from(key: StateKey) -> Self { - Self::Normal(key) - } -} - #[derive(Debug)] pub struct StatePos<'a> { /// Only 2 cases in which the `key` is `None`: @@ -114,7 +108,9 @@ pub trait WindowState: EstimateSize { fn slide_no_output(&mut self) -> Result; } -pub fn create_window_state(call: &WindowFuncCall) -> Result> { +pub type BoxedWindowState = Box; + +pub fn create_window_state(call: &WindowFuncCall) -> Result { assert!(call.frame.bounds.validate().is_ok()); use WindowFuncKind::*; @@ -122,7 +118,7 @@ pub fn create_window_state(call: &WindowFuncCall) -> Result Box::new(rank::RankState::::new(call)), Rank => Box::new(rank::RankState::::new(call)), DenseRank => Box::new(rank::RankState::::new(call)), - Aggregate(_) => Box::new(aggregate::AggregateState::new(call)?), + Aggregate(_) => aggregate::new(call)?, kind => { return Err(ExprError::UnsupportedFunction(format!( "{}({}) -> {}", diff --git a/src/expr/impl/Cargo.toml b/src/expr/impl/Cargo.toml index bee2e4eb10c04..e7b765820dfdd 100644 --- a/src/expr/impl/Cargo.toml +++ b/src/expr/impl/Cargo.toml @@ -25,6 +25,7 @@ chrono = { version = "0.4", default-features = false, features = [ "clock", "std", ] } +chrono-tz = { version = "0.8", features = ["case-insensitive"] } fancy-regex = "0.13" futures-async-stream = { workspace = true } futures-util = "0.3" @@ -32,8 +33,10 @@ hex = "0.4" icelake = { workspace = true } itertools = "0.12" jsonbb = "0.1.2" +linkme = { version = "0.3", features = ["used_linker"] } md5 = "0.7" num-traits = "0.2" +openssl = { version = "0.10", features = ["vendored"] } regex = "1" risingwave_common = { workspace = true } risingwave_expr = { workspace = true } diff --git a/src/expr/impl/benches/expr.rs b/src/expr/impl/benches/expr.rs index fc2ad441cfb96..8468ae86e241b 100644 --- a/src/expr/impl/benches/expr.rs +++ b/src/expr/impl/benches/expr.rs @@ -304,7 +304,7 @@ fn bench_expr(c: &mut Criterion) { } if [ "date_trunc(character varying, timestamp with time zone) -> timestamp with time zone", - "to_timestamp1(character varying, character varying) -> timestamp with time zone", + "char_to_timestamptz(character varying, character varying) -> timestamp with time zone", "to_char(timestamp with time zone, character varying) -> character varying", ] .contains(&format!("{sig:?}").as_str()) @@ -321,12 +321,12 @@ fn bench_expr(c: &mut Criterion) { for (i, t) in sig.inputs_type.iter().enumerate() { use DataType::*; let idx = match (sig.name.as_scalar(), i) { - (PbType::ToTimestamp1, 0) => TIMESTAMP_FORMATTED_STRING, - (PbType::ToChar | PbType::ToTimestamp1, 1) => { + (PbType::CharToTimestamptz, 0) => TIMESTAMP_FORMATTED_STRING, + (PbType::ToChar | PbType::CharToTimestamptz, 1) => { children.push(string_literal("YYYY/MM/DD HH:MM:SS")); continue; } - (PbType::ToChar | PbType::ToTimestamp1, 2) => { + (PbType::ToChar | PbType::CharToTimestamptz, 2) => { children.push(string_literal("Australia/Sydney")); continue; } diff --git a/src/expr/impl/src/lib.rs b/src/expr/impl/src/lib.rs index e9e3cc2b55ad3..a47754cb4e821 100644 --- a/src/expr/impl/src/lib.rs +++ b/src/expr/impl/src/lib.rs @@ -32,6 +32,7 @@ #![feature(test)] #![feature(iter_array_chunks)] #![feature(result_flattening)] +#![feature(used_with_arg)] mod aggregate; mod scalar; diff --git a/src/expr/impl/src/scalar/array_positions.rs b/src/expr/impl/src/scalar/array_positions.rs index 5218dbce6780f..cbae53c001439 100644 --- a/src/expr/impl/src/scalar/array_positions.rs +++ b/src/expr/impl/src/scalar/array_positions.rs @@ -66,10 +66,7 @@ use risingwave_expr::{function, ExprError, Result}; /// 2 /// ``` #[function("array_position(anyarray, any) -> int4")] -fn array_position( - array: Option>, - element: Option>, -) -> Result> { +fn array_position(array: ListRef<'_>, element: Option>) -> Result> { array_position_common(array, element, 0) } @@ -98,7 +95,7 @@ fn array_position( /// ``` #[function("array_position(anyarray, any, int4) -> int4")] fn array_position_start( - array: Option>, + array: ListRef<'_>, element: Option>, start: Option, ) -> Result> { @@ -115,16 +112,15 @@ fn array_position_start( } fn array_position_common( - array: Option>, + array: ListRef<'_>, element: Option>, skip: usize, ) -> Result> { - let Some(left) = array else { return Ok(None) }; - if i32::try_from(left.len()).is_err() { + if i32::try_from(array.len()).is_err() { return Err(ExprError::CastOutOfRange("invalid array length")); } - Ok(left + Ok(array .iter() .skip(skip) .position(|item| item == element) diff --git a/src/expr/impl/src/scalar/array_remove.rs b/src/expr/impl/src/scalar/array_remove.rs index 4608d52602c11..32cfbbae96538 100644 --- a/src/expr/impl/src/scalar/array_remove.rs +++ b/src/expr/impl/src/scalar/array_remove.rs @@ -67,10 +67,6 @@ use risingwave_expr::function; /// select array_remove(ARRAY[array[1],array[2],array[3],array[2],null], array[true]); /// ``` #[function("array_remove(anyarray, any) -> anyarray")] -fn array_remove(array: Option>, elem: Option>) -> Option { - let array = array?; - Some(ListValue::from_datum_iter( - &array.data_type(), - array.iter().filter(|x| x != &elem), - )) +fn array_remove(array: ListRef<'_>, elem: Option>) -> ListValue { + ListValue::from_datum_iter(&array.data_type(), array.iter().filter(|x| x != &elem)) } diff --git a/src/expr/impl/src/scalar/array_replace.rs b/src/expr/impl/src/scalar/array_replace.rs index 5a9014cd445df..637062d946047 100644 --- a/src/expr/impl/src/scalar/array_replace.rs +++ b/src/expr/impl/src/scalar/array_replace.rs @@ -56,16 +56,15 @@ use risingwave_expr::function; /// ``` #[function("array_replace(anyarray, any, any) -> anyarray")] fn array_replace( - array: Option>, + array: ListRef<'_>, elem_from: Option>, elem_to: Option>, -) -> Option { - let array = array?; - Some(ListValue::from_datum_iter( +) -> ListValue { + ListValue::from_datum_iter( &array.data_type(), array.iter().map(|val| match val == elem_from { true => elem_to, false => val, }), - )) + ) } diff --git a/src/expr/impl/src/scalar/case.rs b/src/expr/impl/src/scalar/case.rs index d950cd60af55a..99ec8cb880ddc 100644 --- a/src/expr/impl/src/scalar/case.rs +++ b/src/expr/impl/src/scalar/case.rs @@ -12,12 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::sync::Arc; use risingwave_common::array::{ArrayRef, DataChunk}; use risingwave_common::bail; -use risingwave_common::row::OwnedRow; -use risingwave_common::types::{DataType, Datum}; +use risingwave_common::row::{OwnedRow, Row}; +use risingwave_common::types::{DataType, Datum, ScalarImpl}; use risingwave_expr::expr::{BoxedExpression, Expression}; use risingwave_expr::{build_function, Result}; @@ -106,6 +107,148 @@ impl Expression for CaseExpression { } } +/// With large scale of simple form match arms in case-when expression, +/// we could optimize the `CaseExpression` to `ConstantLookupExpression`, +/// which could significantly facilitate the evaluation of case-when. +#[derive(Debug)] +struct ConstantLookupExpression { + return_type: DataType, + arms: HashMap, + fallback: Option, + /// `operand` must exist at present + operand: BoxedExpression, +} + +impl ConstantLookupExpression { + fn new( + return_type: DataType, + arms: HashMap, + fallback: Option, + operand: BoxedExpression, + ) -> Self { + Self { + return_type, + arms, + fallback, + operand, + } + } + + /// Evaluate the fallback arm with the given input + async fn eval_fallback(&self, input: &OwnedRow) -> Result { + let Some(ref fallback) = self.fallback else { + return Ok(None); + }; + let Ok(res) = fallback.eval_row(input).await else { + bail!("failed to evaluate the input for fallback arm"); + }; + Ok(res) + } + + /// The actual lookup & evaluation logic + /// used in both `eval_row` & `eval` + async fn lookup(&self, datum: Datum, input: &OwnedRow) -> Result { + if datum.is_none() { + return self.eval_fallback(input).await; + } + + if let Some(expr) = self.arms.get(datum.as_ref().unwrap()) { + let Ok(res) = expr.eval_row(input).await else { + bail!("failed to evaluate the input for normal arm"); + }; + Ok(res) + } else { + // Fallback arm goes here + self.eval_fallback(input).await + } + } +} + +#[async_trait::async_trait] +impl Expression for ConstantLookupExpression { + fn return_type(&self) -> DataType { + self.return_type.clone() + } + + async fn eval(&self, input: &DataChunk) -> Result { + let input_len = input.capacity(); + let mut builder = self.return_type().create_array_builder(input_len); + + // Evaluate the input DataChunk at first + let eval_result = self.operand.eval(input).await?; + + for i in 0..input_len { + let datum = eval_result.datum_at(i); + let (row, vis) = input.row_at(i); + + // Check for visibility + if !vis { + builder.append_null(); + continue; + } + + // Note that the `owned_row` here is extracted from input + // rather than from `eval_result` + let owned_row = row.into_owned_row(); + + // Lookup and evaluate the current input datum + if let Ok(datum) = self.lookup(datum, &owned_row).await { + builder.append(datum.as_ref()); + } else { + bail!("failed to lookup and evaluate the expression in `eval`"); + } + } + + Ok(Arc::new(builder.finish())) + } + + async fn eval_row(&self, input: &OwnedRow) -> Result { + let datum = self.operand.eval_row(input).await?; + self.lookup(datum, input).await + } +} + +#[build_function("constant_lookup(...) -> any", type_infer = "panic")] +fn build_constant_lookup_expr( + return_type: DataType, + children: Vec, +) -> Result { + if children.is_empty() { + bail!("children expression must not be empty for constant lookup expression"); + } + + let mut children = children; + + let operand = children.remove(0); + + let mut arms = HashMap::new(); + + // Build the `arms` with iterating over `when` & `then` clauses + let mut iter = children.into_iter().array_chunks(); + for [when, then] in iter.by_ref() { + let Ok(Some(s)) = when.eval_const() else { + bail!("expect when expression to be const"); + }; + arms.insert(s, then); + } + + let fallback = if let Some(else_clause) = iter.into_remainder().unwrap().next() { + if else_clause.return_type() != return_type { + bail!("Type mismatched between else and case."); + } + Some(else_clause) + } else { + None + }; + + Ok(Box::new(ConstantLookupExpression::new( + return_type, + arms, + fallback, + operand, + ))) +} + #[build_function("case(...) -> any", type_infer = "panic")] fn build_case_expr( return_type: DataType, @@ -132,6 +275,7 @@ fn build_case_expr( } else { None }; + Ok(Box::new(CaseExpression::new( return_type, when_clauses, diff --git a/src/expr/impl/src/scalar/cast.rs b/src/expr/impl/src/scalar/cast.rs index dc81e3ab77bac..bf8afc7712f93 100644 --- a/src/expr/impl/src/scalar/cast.rs +++ b/src/expr/impl/src/scalar/cast.rs @@ -87,6 +87,7 @@ pub fn jsonb_to_number>(v: JsonbRef<'_>) -> Result { #[function("cast(int4) -> int2")] #[function("cast(int8) -> int2")] #[function("cast(int8) -> int4")] +#[function("cast(serial) -> int8")] #[function("cast(float4) -> int2")] #[function("cast(float8) -> int2")] #[function("cast(float4) -> int4")] diff --git a/src/expr/impl/src/scalar/encrypt.rs b/src/expr/impl/src/scalar/encrypt.rs new file mode 100644 index 0000000000000..2c7dd89ed980e --- /dev/null +++ b/src/expr/impl/src/scalar/encrypt.rs @@ -0,0 +1,249 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; +use std::sync::LazyLock; + +use openssl::error::ErrorStack; +use openssl::symm::{Cipher, Crypter, Mode as CipherMode}; +use regex::Regex; +use risingwave_expr::{function, CryptographyError, CryptographyStage, ExprError, Result}; + +#[derive(Debug, Clone, PartialEq)] +enum Algorithm { + Aes, +} + +#[derive(Debug, Clone, PartialEq)] +enum Mode { + Cbc, + Ecb, +} +#[derive(Debug, Clone, PartialEq)] +enum Padding { + Pkcs, + None, +} + +#[derive(Clone)] +pub struct CipherConfig { + algorithm: Algorithm, + mode: Mode, + cipher: Cipher, + padding: Padding, + crypt_key: Vec, +} + +/// Because `Cipher` is not `Debug`, we include algorithm, key length and mode manually. +impl std::fmt::Debug for CipherConfig { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CipherConfig") + .field("algorithm", &self.algorithm) + .field("key_len", &self.crypt_key.len()) + .field("mode", &self.mode) + .field("padding", &self.padding) + .finish() + } +} + +static CIPHER_CONFIG_RE: LazyLock = + LazyLock::new(|| Regex::new(r"^(aes)(?:-(cbc|ecb))?(?:/pad:(pkcs|none))?$").unwrap()); + +impl CipherConfig { + fn parse_cipher_config(key: &[u8], input: &str) -> Result { + let Some(caps) = CIPHER_CONFIG_RE.captures(input) else { + return Err(ExprError::InvalidParam { + name: "mode", + reason: format!( + "invalid mode: {}, expect pattern algorithm[-mode][/pad:padding]", + input + ) + .into(), + }); + }; + + let algorithm = match caps.get(1).map(|s| s.as_str()) { + Some("aes") => Algorithm::Aes, + algo => { + return Err(ExprError::InvalidParam { + name: "mode", + reason: format!("expect aes for algorithm, but got: {:?}", algo).into(), + }) + } + }; + + let mode = match caps.get(2).map(|m| m.as_str()) { + Some("cbc") | None => Mode::Cbc, // Default to Cbc if not specified + Some("ecb") => Mode::Ecb, + Some(mode) => { + return Err(ExprError::InvalidParam { + name: "mode", + reason: format!("expect cbc or ecb for mode, but got: {}", mode).into(), + }) + } + }; + + let padding = match caps.get(3).map(|m| m.as_str()) { + Some("pkcs") | None => Padding::Pkcs, // Default to Pkcs if not specified + Some("none") => Padding::None, + Some(padding) => { + return Err(ExprError::InvalidParam { + name: "mode", + reason: format!("expect pkcs or none for padding, but got: {}", padding).into(), + }) + } + }; + + let cipher = match (&algorithm, key.len(), &mode) { + (Algorithm::Aes, 16, Mode::Cbc) => Cipher::aes_128_cbc(), + (Algorithm::Aes, 16, Mode::Ecb) => Cipher::aes_128_ecb(), + (Algorithm::Aes, 24, Mode::Cbc) => Cipher::aes_192_cbc(), + (Algorithm::Aes, 24, Mode::Ecb) => Cipher::aes_192_ecb(), + (Algorithm::Aes, 32, Mode::Cbc) => Cipher::aes_256_cbc(), + (Algorithm::Aes, 32, Mode::Ecb) => Cipher::aes_256_ecb(), + (Algorithm::Aes, n, Mode::Cbc | Mode::Ecb) => { + return Err(ExprError::InvalidParam { + name: "key", + reason: format!("invalid key length: {}, expect 16, 24 or 32", n).into(), + }) + } + }; + + Ok(CipherConfig { + algorithm, + mode, + cipher, + padding, + crypt_key: key.to_vec(), + }) + } + + fn eval(&self, input: &[u8], stage: CryptographyStage) -> Result> { + let operation = match stage { + CryptographyStage::Encrypt => CipherMode::Encrypt, + CryptographyStage::Decrypt => CipherMode::Decrypt, + }; + self.eval_inner(input, operation).map_err(|reason| { + ExprError::Cryptography(Box::new(CryptographyError { stage, reason })) + }) + } + + fn eval_inner( + &self, + input: &[u8], + operation: CipherMode, + ) -> std::result::Result, ErrorStack> { + let mut decrypter = Crypter::new(self.cipher, operation, self.crypt_key.as_ref(), None)?; + let enable_padding = match self.padding { + Padding::Pkcs => true, + Padding::None => false, + }; + decrypter.pad(enable_padding); + let mut decrypt = vec![0; input.len() + self.cipher.block_size()]; + let count = decrypter.update(input, &mut decrypt)?; + let rest = decrypter.finalize(&mut decrypt[count..])?; + decrypt.truncate(count + rest); + Ok(decrypt.into()) + } +} + +/// from [pg doc](https://www.postgresql.org/docs/current/pgcrypto.html#PGCRYPTO-RAW-ENC-FUNCS) +#[function( + "decrypt(bytea, bytea, varchar) -> bytea", + prebuild = "CipherConfig::parse_cipher_config($1, $2)?" +)] +pub fn decrypt(data: &[u8], config: &CipherConfig) -> Result> { + config.eval(data, CryptographyStage::Decrypt) +} + +#[function( + "encrypt(bytea, bytea, varchar) -> bytea", + prebuild = "CipherConfig::parse_cipher_config($1, $2)?" +)] +pub fn encrypt(data: &[u8], config: &CipherConfig) -> Result> { + config.eval(data, CryptographyStage::Encrypt) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_decrypt() { + let data = b"hello world"; + let mode = "aes"; + + let config = CipherConfig::parse_cipher_config( + b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0A\x0B\x0C\x0D\x0E\x0F" as &[u8], + mode, + ) + .unwrap(); + let encrypted = encrypt(data, &config).unwrap(); + + let decrypted = decrypt(&encrypted, &config).unwrap(); + assert_eq!(decrypted, (*data).into()); + } + + #[test] + fn encrypt_testcase() { + let encrypt_wrapper = |data: &[u8], key: &[u8], mode: &str| -> Result> { + let config = CipherConfig::parse_cipher_config(key, mode)?; + encrypt(data, &config) + }; + let decrypt_wrapper = |data: &[u8], key: &[u8], mode: &str| -> Result> { + let config = CipherConfig::parse_cipher_config(key, mode)?; + decrypt(data, &config) + }; + let key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"; + + let encrypted = encrypt_wrapper( + b"\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff", + key, + "aes-ecb/pad:none", + ) + .unwrap(); + + let decrypted = decrypt_wrapper(&encrypted, key, "aes-ecb/pad:none").unwrap(); + assert_eq!( + decrypted, + (*b"\x00\x11\x22\x33\x44\x55\x66\x77\x88\x99\xaa\xbb\xcc\xdd\xee\xff").into() + ) + } + + #[test] + fn test_parse_cipher_config() { + let key = b"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f"; + + let mode_1 = "aes-ecb/pad:none"; + let config = CipherConfig::parse_cipher_config(key, mode_1).unwrap(); + assert_eq!(config.algorithm, Algorithm::Aes); + assert_eq!(config.mode, Mode::Ecb); + assert_eq!(config.padding, Padding::None); + + let mode_2 = "aes-cbc/pad:pkcs"; + let config = CipherConfig::parse_cipher_config(key, mode_2).unwrap(); + assert_eq!(config.algorithm, Algorithm::Aes); + assert_eq!(config.mode, Mode::Cbc); + assert_eq!(config.padding, Padding::Pkcs); + + let mode_3 = "aes"; + let config = CipherConfig::parse_cipher_config(key, mode_3).unwrap(); + assert_eq!(config.algorithm, Algorithm::Aes); + assert_eq!(config.mode, Mode::Cbc); + assert_eq!(config.padding, Padding::Pkcs); + + let mode_4 = "cbc"; + assert!(CipherConfig::parse_cipher_config(key, mode_4).is_err()); + } +} diff --git a/src/expr/impl/src/scalar/external/iceberg.rs b/src/expr/impl/src/scalar/external/iceberg.rs index 3973efee559d6..cd616aa5e475a 100644 --- a/src/expr/impl/src/scalar/external/iceberg.rs +++ b/src/expr/impl/src/scalar/external/iceberg.rs @@ -29,6 +29,7 @@ use risingwave_common::row::OwnedRow; use risingwave_common::types::{DataType, Datum}; use risingwave_expr::expr::BoxedExpression; use risingwave_expr::{build_function, ExprError, Result}; +use thiserror_ext::AsReport; pub struct IcebergTransform { child: BoxedExpression, @@ -93,23 +94,29 @@ fn build(return_type: DataType, mut children: Vec) -> Result varchar")] -pub fn format_type(oid: Option, _typemod: Option) -> Option> { +pub fn format_type(oid: i32, _typemod: Option, writer: &mut impl Write) { // since we don't support type modifier, ignore it. - oid.map(|i| { - DataType::from_oid(i) - .map(|dt| format!("{}", dt).into_boxed_str()) - .unwrap_or("???".into()) - }) + match DataType::from_oid(oid) { + Ok(dt) => write!(writer, "{}", dt).unwrap(), + Err(_) => write!(writer, "???").unwrap(), + } } #[cfg(test)] diff --git a/src/expr/impl/src/scalar/make_time.rs b/src/expr/impl/src/scalar/make_time.rs new file mode 100644 index 0000000000000..ae1ff58033eed --- /dev/null +++ b/src/expr/impl/src/scalar/make_time.rs @@ -0,0 +1,158 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use chrono::{NaiveDate, NaiveDateTime, NaiveTime}; +use risingwave_common::types::{Date, FloatExt, Time, Timestamp, Timestamptz, F64}; +use risingwave_expr::expr_context::TIME_ZONE; +use risingwave_expr::{capture_context, function, ExprError, Result}; + +use crate::scalar::timestamptz::timestamp_at_time_zone; + +pub fn make_naive_date(mut year: i32, month: i32, day: i32) -> Result { + if year == 0 { + return Err(ExprError::InvalidParam { + name: "year, month, day", + reason: format!("invalid date: {}-{}-{}", year, month, day).into(), + }); + } + if year < 0 { + year += 1 + } + NaiveDate::from_ymd_opt(year, month as u32, day as u32).ok_or_else(|| ExprError::InvalidParam { + name: "year, month, day", + reason: format!("invalid date: {}-{}-{}", year, month, day).into(), + }) +} + +fn make_naive_time(hour: i32, min: i32, sec: F64) -> Result { + if !sec.is_finite() || sec.0.is_sign_negative() { + return Err(ExprError::InvalidParam { + name: "sec", + reason: format!("invalid sec: {}", sec).into(), + }); + } + let sec_u32 = sec.0.trunc() as u32; + let microsecond_u32 = ((sec.0 - sec.0.trunc()) * 1_000_000.0).round_ties_even() as u32; + NaiveTime::from_hms_micro_opt(hour as u32, min as u32, sec_u32, microsecond_u32).ok_or_else( + || ExprError::InvalidParam { + name: "hour, min, sec", + reason: format!("invalid time: {}:{}:{}", hour, min, sec).into(), + }, + ) +} + +// year int, month int, day int +#[function("make_date(int4, int4, int4) -> date")] +pub fn make_date(year: i32, month: i32, day: i32) -> Result { + Ok(Date(make_naive_date(year, month, day)?)) +} + +// hour int, min int, sec double precision +#[function("make_time(int4, int4, float8) -> time")] +pub fn make_time(hour: i32, min: i32, sec: F64) -> Result
, Vec, Vec, + Vec, Vec, Vec, Vec, @@ -58,6 +62,8 @@ pub struct DatabaseManager { pub(super) sources: BTreeMap, /// Cached sink information. pub(super) sinks: BTreeMap, + /// Cached subscription information. + pub(super) subscriptions: BTreeMap, /// Cached index information. pub(super) indexes: BTreeMap, /// Cached table information. @@ -83,15 +89,16 @@ pub struct DatabaseManager { impl DatabaseManager { pub async fn new(env: MetaSrvEnv) -> MetaResult { - let databases = Database::list(env.meta_store()).await?; - let schemas = Schema::list(env.meta_store()).await?; - let sources = Source::list(env.meta_store()).await?; - let sinks = Sink::list(env.meta_store()).await?; - let tables = Table::list(env.meta_store()).await?; - let indexes = Index::list(env.meta_store()).await?; - let views = View::list(env.meta_store()).await?; - let functions = Function::list(env.meta_store()).await?; - let connections = Connection::list(env.meta_store()).await?; + let databases = Database::list(env.meta_store_checked()).await?; + let schemas = Schema::list(env.meta_store_checked()).await?; + let sources = Source::list(env.meta_store_checked()).await?; + let sinks = Sink::list(env.meta_store_checked()).await?; + let tables = Table::list(env.meta_store_checked()).await?; + let indexes = Index::list(env.meta_store_checked()).await?; + let views = View::list(env.meta_store_checked()).await?; + let functions = Function::list(env.meta_store_checked()).await?; + let connections = Connection::list(env.meta_store_checked()).await?; + let subscriptions = Subscription::list(env.meta_store_checked()).await?; let mut relation_ref_count = HashMap::new(); @@ -114,6 +121,12 @@ impl DatabaseManager { } (sink.id, sink) })); + let subscriptions = BTreeMap::from_iter(subscriptions.into_iter().map(|subscription| { + for depend_relation_id in &subscription.dependent_relations { + *relation_ref_count.entry(*depend_relation_id).or_default() += 1; + } + (subscription.id, subscription) + })); let indexes = BTreeMap::from_iter(indexes.into_iter().map(|index| (index.id, index))); let tables = BTreeMap::from_iter(tables.into_iter().map(|table| { for depend_relation_id in &table.dependent_relations { @@ -135,6 +148,7 @@ impl DatabaseManager { schemas, sources, sinks, + subscriptions, views, tables, indexes, @@ -168,6 +182,14 @@ impl DatabaseManager { }) .cloned() .collect_vec(), + self.subscriptions + .values() + .filter(|t| { + t.stream_job_status == PbStreamJobStatus::Unspecified as i32 + || t.stream_job_status == PbStreamJobStatus::Created as i32 + }) + .cloned() + .collect_vec(), self.indexes .values() .filter(|t| { @@ -226,6 +248,15 @@ impl DatabaseManager { && x.name.eq(&relation_key.2) }) { Err(MetaError::catalog_duplicated("sink", &relation_key.2)) + } else if self.subscriptions.values().any(|x| { + x.database_id == relation_key.0 + && x.schema_id == relation_key.1 + && x.name.eq(&relation_key.2) + }) { + Err(MetaError::catalog_duplicated( + "subscription", + &relation_key.2, + )) } else if self.views.values().any(|x| { x.database_id == relation_key.0 && x.schema_id == relation_key.1 @@ -298,10 +329,14 @@ impl DatabaseManager { self.sinks.get(&sink_id) } + pub fn get_subscription(&self, subscription_id: SubscriptionId) -> Option<&Subscription> { + self.subscriptions.get(&subscription_id) + } + pub fn get_all_table_options(&self) -> HashMap { self.tables .iter() - .map(|(id, table)| (*id, TableOption::build_table_option(&table.properties))) + .map(|(id, table)| (*id, TableOption::new(table.retention_seconds))) .collect() } @@ -333,6 +368,10 @@ impl DatabaseManager { self.sinks.values().cloned().collect_vec() } + pub fn list_subscriptions(&self) -> Vec { + self.subscriptions.values().cloned().collect_vec() + } + pub fn list_views(&self) -> Vec { self.views.values().cloned().collect_vec() } @@ -358,6 +397,7 @@ impl DatabaseManager { .keys() .copied() .chain(self.sinks.keys().copied()) + .chain(self.subscriptions.keys().copied()) .chain(self.indexes.keys().copied()) .chain(self.sources.keys().copied()) .chain( @@ -396,6 +436,10 @@ impl DatabaseManager { self.tables.values().all(|t| t.schema_id != schema_id) && self.sources.values().all(|s| s.schema_id != schema_id) && self.sinks.values().all(|s| s.schema_id != schema_id) + && self + .subscriptions + .values() + .all(|s| s.schema_id != schema_id) && self.indexes.values().all(|i| i.schema_id != schema_id) && self.views.values().all(|v| v.schema_id != schema_id) } @@ -444,7 +488,7 @@ impl DatabaseManager { } pub fn unmark_creating(&mut self, relation: &RelationKey) { - self.in_progress_creation_tracker.remove(&relation.clone()); + self.in_progress_creation_tracker.remove(relation); } pub fn unmark_creating_streaming_job(&mut self, table_id: TableId) { @@ -533,6 +577,17 @@ impl DatabaseManager { } } + pub fn ensure_subscription_id(&self, subscription_id: SubscriptionId) -> MetaResult<()> { + if self.subscriptions.contains_key(&subscription_id) { + Ok(()) + } else { + Err(MetaError::catalog_id_not_found( + "subscription", + subscription_id, + )) + } + } + pub fn ensure_index_id(&self, index_id: IndexId) -> MetaResult<()> { if self.indexes.contains_key(&index_id) { Ok(()) diff --git a/src/meta/src/manager/catalog/fragment.rs b/src/meta/src/manager/catalog/fragment.rs index 89ea2de7148a7..ecfac68833d0e 100644 --- a/src/meta/src/manager/catalog/fragment.rs +++ b/src/meta/src/manager/catalog/fragment.rs @@ -124,7 +124,7 @@ pub type FragmentManagerRef = Arc; impl FragmentManager { pub async fn new(env: MetaSrvEnv) -> MetaResult { - let table_fragments = TableFragments::list(env.meta_store()).await?; + let table_fragments = TableFragments::list(env.meta_store_checked()).await?; // `expr_context` of `StreamActor` is introduced in 1.6.0. // To ensure compatibility, we fill it for table fragments that were created with older versions. @@ -133,7 +133,7 @@ impl FragmentManager { .map(|tf| (tf.table_id(), tf.fill_expr_context())) .collect(); - let table_revision = TableRevision::get(env.meta_store()).await?; + let table_revision = TableRevision::get(env.meta_store_checked()).await?; Ok(Self { env, @@ -561,7 +561,7 @@ impl FragmentManager { let mut table_fragments = BTreeMapTransaction::new(map); for table_fragment in &to_delete_table_fragments { table_fragments.remove(table_fragment.table_id()); - let backfill_actor_ids = table_fragment.backfill_actor_ids(); + let to_remove_actor_ids: HashSet<_> = table_fragment.actor_ids().into_iter().collect(); let dependent_table_ids = table_fragment.dependent_table_ids(); for (dependent_table_id, _) in dependent_table_ids { if table_ids.contains(&dependent_table_id) { @@ -581,12 +581,11 @@ impl FragmentManager { dependent_table .fragments .values_mut() - .filter(|f| (f.get_fragment_type_mask() & FragmentTypeFlag::Mview as u32) != 0) .flat_map(|f| &mut f.actors) .for_each(|a| { a.dispatcher.retain_mut(|d| { d.downstream_actor_id - .retain(|x| !backfill_actor_ids.contains(x)); + .retain(|x| !to_remove_actor_ids.contains(x)); !d.downstream_actor_id.is_empty() }) }); @@ -722,12 +721,9 @@ impl FragmentManager { Ok(()) } - /// Used in [`crate::barrier::GlobalBarrierManager`], load all actor that need to be sent or + /// Used in [`crate::barrier::GlobalBarrierManager`], load all running actor that need to be sent or /// collected - pub async fn load_all_actors( - &self, - check_state: impl Fn(ActorState, TableId, ActorId) -> bool, - ) -> ActorInfos { + pub async fn load_all_actors(&self) -> ActorInfos { let mut actor_maps = HashMap::new(); let mut barrier_inject_actor_maps = HashMap::new(); @@ -735,7 +731,7 @@ impl FragmentManager { for fragments in map.values() { for (worker_id, actor_states) in fragments.worker_actor_states() { for (actor_id, actor_state) in actor_states { - if check_state(actor_state, fragments.table_id(), actor_id) { + if actor_state == ActorState::Running { actor_maps .entry(worker_id) .or_insert_with(Vec::new) @@ -747,7 +743,7 @@ impl FragmentManager { let barrier_inject_actors = fragments.worker_barrier_inject_actor_states(); for (worker_id, actor_states) in barrier_inject_actors { for (actor_id, actor_state) in actor_states { - if check_state(actor_state, fragments.table_id(), actor_id) { + if actor_state == ActorState::Running { barrier_inject_actor_maps .entry(worker_id) .or_insert_with(Vec::new) @@ -862,6 +858,22 @@ impl FragmentManager { actor_maps } + pub async fn node_actor_count(&self) -> HashMap { + let mut actor_count = HashMap::new(); + + let map = &self.core.read().await.table_fragments; + for fragments in map.values() { + for actor_status in fragments.actor_status.values() { + if let Some(pu) = &actor_status.parallel_unit { + let e = actor_count.entry(pu.worker_node_id).or_insert(0); + *e += 1; + } + } + } + + actor_count + } + // edit the `rate_limit` of the `Source` node in given `source_id`'s fragments // return the actor_ids to be applied pub async fn update_source_rate_limit_by_source_id( @@ -1111,7 +1123,7 @@ impl FragmentManager { let new_created_actors: HashSet<_> = reschedules .values() - .flat_map(|reschedule| reschedule.added_actors.clone()) + .flat_map(|reschedule| reschedule.added_actors.values().flatten().cloned()) .collect(); let to_update_table_fragments = map @@ -1157,7 +1169,7 @@ impl FragmentManager { } = reschedule; // Add actors to this fragment: set the state to `Running`. - for actor_id in added_actors { + for actor_id in added_actors.values().flatten() { table_fragment .actor_status .get_mut(actor_id) @@ -1240,6 +1252,7 @@ impl FragmentManager { } = reschedule; let removed_actor_ids: HashSet<_> = removed_actors.iter().cloned().collect(); + let added_actor_ids = added_actors.values().flatten().cloned().collect_vec(); // Update the dispatcher of the upstream fragments. for (upstream_fragment_id, dispatcher_id) in upstream_fragment_dispatcher_ids { @@ -1272,7 +1285,7 @@ impl FragmentManager { update_actors( dispatcher.downstream_actor_id.as_mut(), &removed_actor_ids, - added_actors, + &added_actor_ids, ); } } @@ -1301,7 +1314,7 @@ impl FragmentManager { update_actors( downstream_actor.upstream_actor_id.as_mut(), &removed_actor_ids, - added_actors, + &added_actor_ids, ); if let Some(node) = downstream_actor.nodes.as_mut() { @@ -1309,7 +1322,7 @@ impl FragmentManager { node, fragment_id, &removed_actor_ids, - added_actors, + &added_actor_ids, ); } } @@ -1319,7 +1332,9 @@ impl FragmentManager { for (table_id, parallelism) in table_parallelism_assignment { if let Some(mut table) = table_fragments.get_mut(table_id) { - table.assigned_parallelism = parallelism; + if table.assigned_parallelism != parallelism { + table.assigned_parallelism = parallelism; + } } } @@ -1492,4 +1507,8 @@ impl FragmentManager { .await .running_fragment_parallelisms(id_filter) } + + pub async fn count_streaming_job(&self) -> usize { + self.core.read().await.table_fragments().len() + } } diff --git a/src/meta/src/manager/catalog/mod.rs b/src/meta/src/manager/catalog/mod.rs index cace146d7fb62..e87c33602d454 100644 --- a/src/meta/src/manager/catalog/mod.rs +++ b/src/meta/src/manager/catalog/mod.rs @@ -36,7 +36,7 @@ use risingwave_connector::source::{should_copy_to_format_encode_options, UPSTREA use risingwave_pb::catalog::table::{OptionalAssociatedSourceId, TableType}; use risingwave_pb::catalog::{ Comment, Connection, CreateType, Database, Function, Index, PbSource, PbStreamJobStatus, - Schema, Sink, Source, StreamJobStatus, Table, View, + Schema, Sink, Source, StreamJobStatus, Subscription, Table, View, }; use risingwave_pb::ddl_service::{alter_owner_request, alter_set_schema_request}; use risingwave_pb::meta::subscribe_response::{Info, Operation}; @@ -58,6 +58,7 @@ pub type SchemaId = u32; pub type TableId = u32; pub type SourceId = u32; pub type SinkId = u32; +pub type SubscriptionId = u32; pub type RelationId = u32; pub type IndexId = u32; pub type ViewId = u32; @@ -71,6 +72,7 @@ pub enum RelationIdEnum { Index(IndexId), View(ViewId), Sink(SinkId), + Subscription(SubscriptionId), Source(SourceId), } @@ -87,7 +89,7 @@ macro_rules! commit_meta_with_trx { $val_txn.apply_to_txn(&mut $trx).await?; )* // Commit to meta store - $manager.env.meta_store().txn($trx).await?; + $manager.env.meta_store_checked().txn($trx).await?; // Upon successful commit, commit the change to in-mem meta $( $val_txn.commit(); @@ -314,6 +316,7 @@ impl CatalogManager { let mut schemas = BTreeMapTransaction::new(&mut database_core.schemas); let mut sources = BTreeMapTransaction::new(&mut database_core.sources); let mut sinks = BTreeMapTransaction::new(&mut database_core.sinks); + let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions); let mut tables = BTreeMapTransaction::new(&mut database_core.tables); let mut indexes = BTreeMapTransaction::new(&mut database_core.indexes); let mut views = BTreeMapTransaction::new(&mut database_core.views); @@ -346,6 +349,7 @@ impl CatalogManager { let schemas_to_drop = drop_by_database_id!(schemas, database_id); let sources_to_drop = drop_by_database_id!(sources, database_id); let sinks_to_drop = drop_by_database_id!(sinks, database_id); + let subscriptions_to_drop = drop_by_database_id!(subscriptions, database_id); let tables_to_drop = drop_by_database_id!(tables, database_id); let indexes_to_drop = drop_by_database_id!(indexes, database_id); let views_to_drop = drop_by_database_id!(views, database_id); @@ -380,6 +384,7 @@ impl CatalogManager { schemas, sources, sinks, + subscriptions, tables, indexes, views, @@ -392,6 +397,11 @@ impl CatalogManager { .chain(schemas_to_drop.iter().map(|schema| schema.owner)) .chain(sources_to_drop.iter().map(|source| source.owner)) .chain(sinks_to_drop.iter().map(|sink| sink.owner)) + .chain( + subscriptions_to_drop + .iter() + .map(|subscription| subscription.owner), + ) .chain( tables_to_drop .iter() @@ -441,6 +451,11 @@ impl CatalogManager { .into_iter() .map(|sink| StreamingJobId::new(sink.id)), ) + .chain( + subscriptions_to_drop + .into_iter() + .map(|subscription| StreamingJobId::new(subscription.id)), + ) .collect_vec(); let source_deleted_ids = sources_to_drop .into_iter() @@ -696,6 +711,9 @@ impl CatalogManager { .await } StreamingJob::Sink(sink, _) => self.start_create_sink_procedure(sink).await, + StreamingJob::Subscription(subscription) => { + self.start_create_subscription_procedure(subscription).await + } StreamingJob::Index(index, index_table) => { self.start_create_index_procedure(index, index_table).await } @@ -797,13 +815,15 @@ impl CatalogManager { Ok(()) } - fn assert_table_creating(tables: &BTreeMap, table: &Table) { - if let Some(t) = tables.get(&table.id) - && let Ok(StreamJobStatus::Creating) = t.get_stream_job_status() - { + fn check_table_creating(tables: &BTreeMap, table: &Table) -> MetaResult<()> { + return if let Some(t) = tables.get(&table.id) { + assert_eq!(t.get_stream_job_status(), Ok(StreamJobStatus::Creating)); + Ok(()) } else { - panic!("Table must be in creating procedure: {table:#?}") - } + // If the table does not exist, it should be created in Foreground and cleaned during recovery in some cases. + assert_eq!(table.create_type(), CreateType::Foreground); + Err(anyhow!("failed to create table during recovery").into()) + }; } pub async fn assert_tables_deleted(&self, table_ids: Vec) { @@ -1007,7 +1027,7 @@ impl CatalogManager { let database_core = &mut core.database; let tables = &mut database_core.tables; if cfg!(not(test)) { - Self::assert_table_creating(tables, &table); + Self::check_table_creating(tables, &table)?; } let mut tables = BTreeMapTransaction::new(tables); @@ -1120,6 +1140,7 @@ impl CatalogManager { let mut tables = BTreeMapTransaction::new(&mut database_core.tables); let mut sources = BTreeMapTransaction::new(&mut database_core.sources); let mut sinks = BTreeMapTransaction::new(&mut database_core.sinks); + let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions); let mut views = BTreeMapTransaction::new(&mut database_core.views); let mut users = BTreeMapTransaction::new(&mut user_core.user_info); @@ -1143,6 +1164,7 @@ impl CatalogManager { let mut all_internal_table_ids: HashSet = HashSet::default(); let mut all_index_ids: HashSet = HashSet::default(); let mut all_sink_ids: HashSet = HashSet::default(); + let mut all_subscription_ids: HashSet = HashSet::default(); let mut all_source_ids: HashSet = HashSet::default(); let mut all_view_ids: HashSet = HashSet::default(); let mut all_cdc_source_ids: HashSet = HashSet::default(); @@ -1172,6 +1194,18 @@ impl CatalogManager { }) .collect_vec(); + let subscriptions_depend_on = subscriptions + .tree_ref() + .iter() + .filter_map(|(_, subscription)| { + if subscription.dependent_relations.contains(&relation_id) { + Some(RelationInfo::Subscription(subscription.clone())) + } else { + None + } + }) + .collect_vec(); + let views_depend_on = views .tree_ref() .iter() @@ -1188,6 +1222,7 @@ impl CatalogManager { tables_depend_on .into_iter() .chain(sinks_depend_on) + .chain(subscriptions_depend_on) .chain(views_depend_on) .collect() }; @@ -1225,6 +1260,14 @@ impl CatalogManager { bail!("sink doesn't exist"); } } + RelationIdEnum::Subscription(subscription_id) => { + let subscription = subscriptions.get(&subscription_id).cloned(); + if let Some(subscription) = subscription { + deque.push_back(RelationInfo::Subscription(subscription)); + } else { + bail!("subscription doesn't exist"); + } + } RelationIdEnum::View(view_id) => { let view = views.get(&view_id).cloned(); if let Some(view) = view { @@ -1499,6 +1542,41 @@ impl CatalogManager { } } } + RelationInfo::Subscription(subscription) => { + if !all_subscription_ids.insert(subscription.id) { + continue; + } + let table_fragments = fragment_manager + .select_table_fragments_by_table_id(&subscription.id.into()) + .await?; + + all_internal_table_ids.extend(table_fragments.internal_table_ids()); + + if let Some(ref_count) = database_core + .relation_ref_count + .get(&subscription.id) + .cloned() + { + if ref_count > 0 { + // Other relations depend on it. + match drop_mode { + DropMode::Restrict => { + return Err(MetaError::permission_denied(format!( + "Fail to delete subscription `{}` because {} other relation(s) depend on it", + subscription.name, ref_count + ))); + } + DropMode::Cascade => { + for relation_info in + relations_depend_on(subscription.id as RelationId) + { + deque.push_back(relation_info); + } + } + } + } + } + } } } @@ -1526,6 +1604,10 @@ impl CatalogManager { .iter() .map(|sink_id| sinks.remove(*sink_id).unwrap()) .collect_vec(); + let subscriptions_removed = all_subscription_ids + .iter() + .map(|subscription_id| subscriptions.remove(*subscription_id).unwrap()) + .collect_vec(); if !matches!(relation, RelationIdEnum::Sink(_)) { let table_sinks = sinks_removed @@ -1584,7 +1666,16 @@ impl CatalogManager { ) }; - commit_meta!(self, tables, indexes, sources, views, sinks, users)?; + commit_meta!( + self, + tables, + indexes, + sources, + views, + sinks, + users, + subscriptions + )?; for index in &indexes_removed { user_core.decrease_ref(index.owner); @@ -1608,6 +1699,10 @@ impl CatalogManager { user_core.decrease_ref(sink.owner); } + for subscription in &subscriptions_removed { + user_core.decrease_ref(subscription.owner); + } + for user in users_need_update { self.notify_frontend(Operation::Update, Info::User(user)) .await; @@ -1636,6 +1731,12 @@ impl CatalogManager { } } + for subscription in &subscriptions_removed { + for dependent_relation_id in &subscription.dependent_relations { + database_core.decrease_ref_count(*dependent_relation_id); + } + } + let version = self .notify_frontend( Operation::Delete, @@ -1660,6 +1761,13 @@ impl CatalogManager { .chain(sinks_removed.into_iter().map(|sink| Relation { relation_info: RelationInfo::Sink(sink).into(), })) + .chain( + subscriptions_removed + .into_iter() + .map(|subscription| Relation { + relation_info: RelationInfo::Subscription(subscription).into(), + }), + ) .collect_vec(), }), ) @@ -1669,6 +1777,7 @@ impl CatalogManager { .into_iter() .map(|id| id.into()) .chain(all_sink_ids.into_iter().map(|id| id.into())) + .chain(all_subscription_ids.into_iter().map(|id| id.into())) .chain(all_cdc_source_ids.into_iter().map(|id| id.into())) .collect_vec(); @@ -1714,6 +1823,7 @@ impl CatalogManager { vec![table], vec![], vec![], + vec![], source, ) .await @@ -1730,6 +1840,7 @@ impl CatalogManager { mut to_update_tables: Vec
, mut to_update_views: Vec, mut to_update_sinks: Vec, + mut to_update_subscriptions: Vec, to_update_source: Option, ) -> MetaResult { for table in database_mgr.tables.values() { @@ -1749,17 +1860,30 @@ impl CatalogManager { } for sink in database_mgr.sinks.values() { - if sink.dependent_relations.contains(&relation_id) { + if sink.dependent_relations.contains(&relation_id) + || sink.target_table == Some(relation_id) + { let mut sink = sink.clone(); sink.definition = alter_relation_rename_refs(&sink.definition, from, to); to_update_sinks.push(sink); } } + for subscription in database_mgr.subscriptions.values() { + if subscription.dependent_relations.contains(&relation_id) { + let mut subscription = subscription.clone(); + subscription.definition = + alter_relation_rename_refs(&subscription.definition, from, to); + to_update_subscriptions.push(subscription); + } + } + // commit meta. let mut tables = BTreeMapTransaction::new(&mut database_mgr.tables); let mut views = BTreeMapTransaction::new(&mut database_mgr.views); let mut sinks = BTreeMapTransaction::new(&mut database_mgr.sinks); + let mut subscriptions: BTreeMapTransaction<'_, u32, risingwave_pb::catalog::Subscription> = + BTreeMapTransaction::new(&mut database_mgr.subscriptions); let mut sources = BTreeMapTransaction::new(&mut database_mgr.sources); to_update_tables.iter().for_each(|table| { tables.insert(table.id, table.clone()); @@ -1770,6 +1894,9 @@ impl CatalogManager { to_update_sinks.iter().for_each(|sink| { sinks.insert(sink.id, sink.clone()); }); + to_update_subscriptions.iter().for_each(|subscription| { + subscriptions.insert(subscription.id, subscription.clone()); + }); if let Some(source) = &to_update_source { sources.insert(source.id, source.clone()); } @@ -1780,6 +1907,7 @@ impl CatalogManager { !to_update_tables.is_empty() || !to_update_views.is_empty() || !to_update_sinks.is_empty() + || !to_update_subscriptions.is_empty() || to_update_source.is_some() ); let version = self @@ -1797,6 +1925,13 @@ impl CatalogManager { .chain(to_update_sinks.into_iter().map(|sink| Relation { relation_info: RelationInfo::Sink(sink).into(), })) + .chain( + to_update_subscriptions + .into_iter() + .map(|subscription| Relation { + relation_info: RelationInfo::Subscription(subscription).into(), + }), + ) .chain(to_update_source.into_iter().map(|source| Relation { relation_info: RelationInfo::Source(source).into(), })) @@ -1839,6 +1974,7 @@ impl CatalogManager { vec![], vec![view], vec![], + vec![], None, ) .await @@ -1877,6 +2013,47 @@ impl CatalogManager { Ok(version) } + pub async fn alter_subscription_name( + &self, + subscription_id: SubscriptionId, + subscription_name: &str, + ) -> MetaResult { + let core = &mut *self.core.lock().await; + let database_core = &mut core.database; + database_core.ensure_subscription_id(subscription_id)?; + + // 1. validate new subscription name. + let mut subscription = database_core + .subscriptions + .get(&subscription_id) + .unwrap() + .clone(); + database_core.check_relation_name_duplicated(&( + subscription.database_id, + subscription.schema_id, + subscription_name.to_string(), + ))?; + + // 2. rename subscription and its definition. + subscription.name = subscription_name.to_string(); + subscription.definition = + alter_relation_rename(&subscription.definition, subscription_name); + + // 3. commit meta. + let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions); + subscriptions.insert(subscription_id, subscription.clone()); + commit_meta!(self, subscriptions)?; + + let version = self + .notify_frontend_relation_info( + Operation::Update, + RelationInfo::Subscription(subscription), + ) + .await; + + Ok(version) + } + pub async fn alter_source_name( &self, source_id: SourceId, @@ -1908,6 +2085,7 @@ impl CatalogManager { vec![], vec![], vec![], + vec![], Some(source), ) .await @@ -2211,6 +2389,40 @@ impl CatalogManager { user_core.increase_ref(owner_id); user_core.decrease_ref(old_owner_id); } + alter_owner_request::Object::SubscriptionId(subscription_id) => { + database_core.ensure_subscription_id(subscription_id)?; + let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions); + let mut tables = BTreeMapTransaction::new(&mut database_core.tables); + let mut subscription = subscriptions.get_mut(subscription_id).unwrap(); + let old_owner_id = subscription.owner; + if old_owner_id == owner_id { + return Ok(IGNORED_NOTIFICATION_VERSION); + } + subscription.owner = owner_id; + + let mut relations = vec![Relation { + relation_info: Some(RelationInfo::Subscription(subscription.clone())), + }]; + + // internal tables + let internal_table_ids = fragment_manager + .select_table_fragments_by_table_id(&(subscription_id.into())) + .await? + .internal_table_ids(); + for id in internal_table_ids { + let mut table = tables.get_mut(id).unwrap(); + assert_eq!(old_owner_id, table.owner); + table.owner = owner_id; + relations.push(Relation { + relation_info: Some(RelationInfo::Table(table.clone())), + }); + } + + relation_info = Info::RelationGroup(RelationGroup { relations }); + commit_meta!(self, subscriptions, tables)?; + user_core.increase_ref(owner_id); + user_core.decrease_ref(old_owner_id); + } }; let version = self.notify_frontend(Operation::Update, relation_info).await; @@ -2435,6 +2647,42 @@ impl CatalogManager { let version = self.notify_frontend(Operation::Update, notify_info).await; return Ok(version); } + alter_set_schema_request::Object::SubscriptionId(subscription_id) => { + database_core.ensure_subscription_id(subscription_id)?; + let Subscription { + name, schema_id, .. + } = database_core.subscriptions.get(&subscription_id).unwrap(); + if *schema_id == new_schema_id { + return Ok(IGNORED_NOTIFICATION_VERSION); + } + + // internal tables. + let to_update_internal_table_ids = Vec::from_iter( + fragment_manager + .select_table_fragments_by_table_id(&(subscription_id.into())) + .await? + .internal_table_ids(), + ); + + database_core.check_relation_name_duplicated(&( + database_id, + new_schema_id, + name.to_owned(), + ))?; + let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions); + let mut subscription = subscriptions.get_mut(subscription_id).unwrap(); + subscription.schema_id = new_schema_id; + relation_infos.push(Some(RelationInfo::Subscription(subscription.clone()))); + + let mut tables = BTreeMapTransaction::new(&mut database_core.tables); + for table_id in to_update_internal_table_ids { + let mut table = tables.get_mut(table_id).unwrap(); + table.schema_id = new_schema_id; + relation_infos.push(Some(RelationInfo::Table(table.clone()))); + } + + commit_meta!(self, subscriptions, tables)?; + } } let version = self @@ -2934,6 +3182,114 @@ impl CatalogManager { } } + pub async fn start_create_subscription_procedure( + &self, + subscription: &Subscription, + ) -> MetaResult<()> { + let core = &mut *self.core.lock().await; + let database_core = &mut core.database; + let user_core = &mut core.user; + database_core.ensure_database_id(subscription.database_id)?; + database_core.ensure_schema_id(subscription.schema_id)?; + for dependent_id in &subscription.dependent_relations { + database_core.ensure_table_view_or_source_id(dependent_id)?; + } + let key = ( + subscription.database_id, + subscription.schema_id, + subscription.name.clone(), + ); + database_core.check_relation_name_duplicated(&key)?; + #[cfg(not(test))] + user_core.ensure_user_id(subscription.owner)?; + + if database_core.has_in_progress_creation(&key) { + bail!("subscription already in creating procedure"); + } else { + database_core.mark_creating(&key); + database_core.mark_creating_streaming_job(subscription.id, key); + for &dependent_relation_id in &subscription.dependent_relations { + database_core.increase_ref_count(dependent_relation_id); + } + user_core.increase_ref(subscription.owner); + Ok(()) + } + } + + pub async fn finish_create_subscription_procedure( + &self, + mut internal_tables: Vec
, + mut subscription: Subscription, + ) -> MetaResult { + let core = &mut *self.core.lock().await; + let database_core = &mut core.database; + let key = ( + subscription.database_id, + subscription.schema_id, + subscription.name.clone(), + ); + let mut tables = BTreeMapTransaction::new(&mut database_core.tables); + let mut subscriptions = BTreeMapTransaction::new(&mut database_core.subscriptions); + assert!( + !subscriptions.contains_key(&subscription.id) + && database_core.in_progress_creation_tracker.contains(&key), + "subscription must be in creating procedure" + ); + + database_core.in_progress_creation_tracker.remove(&key); + database_core + .in_progress_creation_streaming_job + .remove(&subscription.id); + + subscription.stream_job_status = PbStreamJobStatus::Created.into(); + subscriptions.insert(subscription.id, subscription.clone()); + for table in &mut internal_tables { + table.stream_job_status = PbStreamJobStatus::Created.into(); + tables.insert(table.id, table.clone()); + } + commit_meta!(self, subscriptions, tables)?; + + let version = self + .notify_frontend( + Operation::Add, + Info::RelationGroup(RelationGroup { + relations: vec![Relation { + relation_info: RelationInfo::Subscription(subscription.to_owned()).into(), + }] + .into_iter() + .chain(internal_tables.into_iter().map(|internal_table| Relation { + relation_info: RelationInfo::Table(internal_table).into(), + })) + .collect_vec(), + }), + ) + .await; + + Ok(version) + } + + pub async fn cancel_create_subscription_procedure(&self, subscription: &Subscription) { + let core = &mut *self.core.lock().await; + let database_core = &mut core.database; + let user_core = &mut core.user; + let key = ( + subscription.database_id, + subscription.schema_id, + subscription.name.clone(), + ); + assert!( + !database_core.subscriptions.contains_key(&subscription.id), + "subscription must be in creating procedure" + ); + + database_core.unmark_creating(&key); + database_core.unmark_creating_streaming_job(subscription.id); + for &dependent_relation_id in &subscription.dependent_relations { + database_core.decrease_ref_count(dependent_relation_id); + } + user_core.decrease_ref(subscription.owner); + } + /// This is used for `ALTER TABLE ADD/DROP COLUMN`. pub async fn start_replace_table_procedure(&self, stream_job: &StreamingJob) -> MetaResult<()> { let StreamingJob::Table(source, table, ..) = stream_job else { @@ -3019,7 +3375,7 @@ impl CatalogManager { let mut updated_indexes = vec![]; - if let Some(table_col_index_mapping) = table_col_index_mapping.clone() { + if let Some(table_col_index_mapping) = table_col_index_mapping { let expr_rewriter = ReplaceTableExprRewriter { table_col_index_mapping, }; @@ -3233,6 +3589,10 @@ impl CatalogManager { self.core.lock().await.database.list_sinks() } + pub async fn list_subscriptions(&self) -> Vec { + self.core.lock().await.database.list_subscriptions() + } + pub async fn list_views(&self) -> Vec { self.core.lock().await.database.list_views() } @@ -3372,6 +3732,11 @@ impl CatalogManager { self.core.lock().await.database.sinks.len() } + #[cfg_attr(coverage, coverage(off))] + pub async fn subscription_count(&self) -> usize { + self.core.lock().await.database.subscriptions.len() + } + #[cfg_attr(coverage, coverage(off))] pub async fn function_count(&self) -> usize { self.core.lock().await.database.functions.len() @@ -3397,7 +3762,7 @@ impl CatalogManager { ..Default::default() }; - default_user.insert(self.env.meta_store()).await?; + default_user.insert(self.env.meta_store_checked()).await?; core.user_info.insert(default_user.id, default_user); } } diff --git a/src/meta/src/manager/catalog/user.rs b/src/meta/src/manager/catalog/user.rs index 851016ec0e603..d148e4c591819 100644 --- a/src/meta/src/manager/catalog/user.rs +++ b/src/meta/src/manager/catalog/user.rs @@ -35,7 +35,7 @@ pub struct UserManager { impl UserManager { pub async fn new(env: MetaSrvEnv, database: &DatabaseManager) -> MetaResult { - let users = UserInfo::list(env.meta_store()).await?; + let users = UserInfo::list(env.meta_store_checked()).await?; let user_info = BTreeMap::from_iter(users.into_iter().map(|user| (user.id, user))); let mut user_manager = Self { @@ -53,6 +53,12 @@ impl UserManager { .chain(database.sources.values().map(|source| source.owner)) .chain(database.sinks.values().map(|sink| sink.owner)) .chain(database.indexes.values().map(|index| index.owner)) + .chain( + database + .subscriptions + .values() + .map(|subscriptions| subscriptions.owner), + ) .chain( database .tables diff --git a/src/meta/src/manager/catalog/utils.rs b/src/meta/src/manager/catalog/utils.rs index b9272b533daac..69592180aa60a 100644 --- a/src/meta/src/manager/catalog/utils.rs +++ b/src/meta/src/manager/catalog/utils.rs @@ -107,6 +107,7 @@ pub fn alter_relation_rename_refs(definition: &str, from: &str, to: &str) -> Str stmt: CreateSinkStatement { sink_from: CreateSink::AsQuery(query), + into_table_name: None, .. }, } => { @@ -117,9 +118,27 @@ pub fn alter_relation_rename_refs(definition: &str, from: &str, to: &str) -> Str stmt: CreateSinkStatement { sink_from: CreateSink::From(table_name), + into_table_name: None, .. }, } => replace_table_name(table_name, to), + Statement::CreateSink { + stmt: CreateSinkStatement { + sink_from, + into_table_name: Some(table_name), + .. + } + } => { + let idx = table_name.0.len() - 1; + if table_name.0[idx].real_value() == from { + table_name.0[idx] = Ident::new_unchecked(to); + } else { + match sink_from { + CreateSink::From(table_name) => replace_table_name(table_name, to), + CreateSink::AsQuery(query) => QueryRewriter::rewrite_query(query, from, to), + } + } + } _ => unreachable!(), }; stmt.to_string() diff --git a/src/meta/src/manager/cluster.rs b/src/meta/src/manager/cluster.rs index 357c4a4cc887a..b6c31ddb51688 100644 --- a/src/meta/src/manager/cluster.rs +++ b/src/meta/src/manager/cluster.rs @@ -31,6 +31,8 @@ use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty; use risingwave_pb::meta::heartbeat_request; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::meta::update_worker_node_schedulability_request::Schedulability; +use thiserror_ext::AsReport; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; use tokio::sync::oneshot::Sender; use tokio::sync::{RwLock, RwLockReadGuard}; use tokio::task::JoinHandle; @@ -106,11 +108,12 @@ impl ClusterManager { property: AddNodeProperty, resource: risingwave_pb::common::worker_node::Resource, ) -> MetaResult { - let worker_node_parallelism = property.worker_node_parallelism as usize; + let new_worker_parallelism = property.worker_node_parallelism as usize; let mut property = self.parse_property(r#type, property); let mut core = self.core.write().await; if let Some(worker) = core.get_worker_by_host_mut(host_address.clone()) { + tracing::info!("worker {} re-joined the cluster", worker.worker_id()); worker.update_resource(Some(resource)); worker.update_started_at(timestamp_now_sec()); if let Some(property) = &mut property { @@ -122,8 +125,8 @@ impl ClusterManager { .unwrap_or_default(); } - let current_parallelism = worker.worker_node.parallel_units.len(); - if current_parallelism == worker_node_parallelism + let old_worker_parallelism = worker.worker_node.parallel_units.len(); + if old_worker_parallelism == new_worker_parallelism && worker.worker_node.property == property { worker.update_expire_at(self.max_heartbeat_interval); @@ -131,31 +134,45 @@ impl ClusterManager { } let mut new_worker = worker.clone(); - match current_parallelism.cmp(&worker_node_parallelism) { + match old_worker_parallelism.cmp(&new_worker_parallelism) { Ordering::Less => { tracing::info!( "worker {} parallelism updated from {} to {}", new_worker.worker_node.id, - current_parallelism, - worker_node_parallelism + old_worker_parallelism, + new_worker_parallelism ); let parallel_units = self .generate_cn_parallel_units( - worker_node_parallelism - current_parallelism, + new_worker_parallelism - old_worker_parallelism, new_worker.worker_id(), ) .await?; new_worker.worker_node.parallel_units.extend(parallel_units); } Ordering::Greater => { - // Warn and keep the original parallelism if the worker registered with a - // smaller parallelism. - tracing::warn!( - "worker {} parallelism is less than current, current is {}, but received {}", - new_worker.worker_id(), - current_parallelism, - worker_node_parallelism - ); + if !self.env.opts.disable_automatic_parallelism_control { + // Handing over to the subsequent recovery loop for a forced reschedule. + tracing::info!( + "worker {} parallelism reduced from {} to {}", + new_worker.worker_node.id, + old_worker_parallelism, + new_worker_parallelism + ); + new_worker + .worker_node + .parallel_units + .truncate(new_worker_parallelism) + } else { + // Warn and keep the original parallelism if the worker registered with a + // smaller parallelism, entering compatibility mode. + tracing::warn!( + "worker {} parallelism is less than current, current is {}, but received {}", + new_worker.worker_id(), + new_worker_parallelism, + old_worker_parallelism, + ); + } } Ordering::Equal => {} } @@ -171,7 +188,7 @@ impl ClusterManager { } new_worker.update_expire_at(self.max_heartbeat_interval); - new_worker.insert(self.env.meta_store()).await?; + new_worker.insert(self.env.meta_store_checked()).await?; *worker = new_worker; return Ok(worker.to_protobuf()); } @@ -192,7 +209,7 @@ impl ClusterManager { // Generate parallel units. let parallel_units = if r#type == WorkerType::ComputeNode { - self.generate_cn_parallel_units(worker_node_parallelism, worker_id) + self.generate_cn_parallel_units(new_worker_parallelism, worker_id) .await? } else { vec![] @@ -215,18 +232,31 @@ impl ClusterManager { worker.update_started_at(timestamp_now_sec()); worker.update_resource(Some(resource)); // Persist worker node. - worker.insert(self.env.meta_store()).await?; + worker.insert(self.env.meta_store_checked()).await?; // Update core. core.add_worker_node(worker); + + tracing::info!( + "new worker {} from {}:{} joined cluster", + worker_id, + host_address.get_host(), + host_address.get_port() + ); + Ok(worker_node) } pub async fn activate_worker_node(&self, host_address: HostAddress) -> MetaResult<()> { let mut core = self.core.write().await; let mut worker = core.get_worker_by_host_checked(host_address.clone())?; + + let worker_id = worker.worker_id(); + + tracing::info!("worker {} activating", worker_id); + if worker.worker_node.state != State::Running as i32 { worker.worker_node.state = State::Running as i32; - worker.insert(self.env.meta_store()).await?; + worker.insert(self.env.meta_store_checked()).await?; core.update_worker_node(worker.clone()); } @@ -243,6 +273,8 @@ impl ClusterManager { .notify_local_subscribers(LocalNotification::WorkerNodeActivated(worker.worker_node)) .await; + tracing::info!("worker {} activated", worker_id); + Ok(()) } @@ -277,7 +309,7 @@ impl ClusterManager { } } - self.env.meta_store().txn(txn).await?; + self.env.meta_store_checked().txn(txn).await?; for var_txn in var_txns { var_txn.commit(); @@ -293,7 +325,7 @@ impl ClusterManager { let worker_node = worker.to_protobuf(); // Persist deletion. - Worker::delete(self.env.meta_store(), &host_address).await?; + Worker::delete(self.env.meta_store_checked(), &host_address).await?; // Update core. core.delete_worker_node(worker); @@ -400,11 +432,11 @@ impl ClusterManager { } Err(err) => { tracing::warn!( - "Failed to delete expired worker {} {:#?}, current timestamp {}. {:?}", + error = %err.as_report(), + "Failed to delete expired worker {} {:#?}, current timestamp {}", worker_id, key, now, - err, ); } } @@ -427,6 +459,22 @@ impl ClusterManager { core.list_worker_node(worker_type, worker_state) } + pub async fn subscribe_active_streaming_compute_nodes( + &self, + ) -> (Vec, UnboundedReceiver) { + let core = self.core.read().await; + let worker_nodes = core.list_streaming_worker_node(Some(State::Running)); + let (tx, rx) = unbounded_channel(); + + // insert before release the read lock to ensure that we don't lose any update in between + self.env + .notification_manager() + .insert_local_sender(tx) + .await; + drop(core); + (worker_nodes, rx) + } + /// A convenient method to get all running compute nodes that may have running actors on them /// i.e. CNs which are running pub async fn list_active_streaming_compute_nodes(&self) -> Vec { @@ -434,11 +482,6 @@ impl ClusterManager { core.list_streaming_worker_node(Some(State::Running)) } - pub async fn list_active_streaming_parallel_units(&self) -> Vec { - let core = self.core.read().await; - core.list_active_streaming_parallel_units() - } - /// Get the cluster info used for scheduling a streaming job, containing all nodes that are /// running and schedulable pub async fn list_active_serving_compute_nodes(&self) -> Vec { @@ -521,7 +564,7 @@ impl ClusterManagerCore { pub const MAX_WORKER_REUSABLE_ID_COUNT: usize = 1 << Self::MAX_WORKER_REUSABLE_ID_BITS; async fn new(env: MetaSrvEnv) -> MetaResult { - let meta_store = env.meta_store(); + let meta_store = env.meta_store_checked(); let mut workers = Worker::list(meta_store).await?; let used_transactional_ids: HashSet<_> = workers @@ -549,7 +592,7 @@ impl ClusterManagerCore { None => { return Err(MetaError::unavailable( "no available transactional id for worker", - )) + )); } Some(id) => id, }; @@ -672,13 +715,6 @@ impl ClusterManagerCore { .collect() } - fn list_active_streaming_parallel_units(&self) -> Vec { - self.list_streaming_worker_node(Some(State::Running)) - .into_iter() - .flat_map(|w| w.parallel_units) - .collect() - } - // Lists active worker nodes fn get_streaming_cluster_info(&self) -> StreamingClusterInfo { let mut streaming_worker_node = self.list_streaming_worker_node(Some(State::Running)); @@ -775,8 +811,11 @@ mod tests { async fn test_cluster_manager() -> MetaResult<()> { let env = MetaSrvEnv::for_test().await; - let cluster_manager = - Arc::new(ClusterManager::new(env, Duration::new(0, 0)).await.unwrap()); + let cluster_manager = Arc::new( + ClusterManager::new(env.clone(), Duration::new(0, 0)) + .await + .unwrap(), + ); let mut worker_nodes = Vec::new(); let worker_count = 5usize; @@ -863,8 +902,15 @@ mod tests { ) .await .unwrap(); - assert_eq!(worker_node.parallel_units.len(), fake_parallelism + 4); - assert_cluster_manager(&cluster_manager, parallel_count + 4).await; + + if !env.opts.disable_automatic_parallelism_control { + assert_eq!(worker_node.parallel_units.len(), fake_parallelism - 2); + assert_cluster_manager(&cluster_manager, parallel_count - 2).await; + } else { + // compatibility mode + assert_eq!(worker_node.parallel_units.len(), fake_parallelism + 4); + assert_cluster_manager(&cluster_manager, parallel_count + 4).await; + } let worker_to_delete_count = 4usize; for i in 0..worker_to_delete_count { @@ -928,7 +974,12 @@ mod tests { } async fn assert_cluster_manager(cluster_manager: &ClusterManager, parallel_count: usize) { - let parallel_units = cluster_manager.list_active_streaming_parallel_units().await; + let parallel_units = cluster_manager + .list_active_serving_compute_nodes() + .await + .into_iter() + .flat_map(|w| w.parallel_units) + .collect_vec(); assert_eq!(parallel_units.len(), parallel_count); } diff --git a/src/meta/src/manager/diagnose.rs b/src/meta/src/manager/diagnose.rs index a8c1d050ab748..76184de9eb7ce 100644 --- a/src/meta/src/manager/diagnose.rs +++ b/src/meta/src/manager/diagnose.rs @@ -13,7 +13,7 @@ // limitations under the License. use std::cmp::{Ordering, Reverse}; -use std::collections::{BinaryHeap, HashMap}; +use std::collections::BinaryHeap; use std::fmt::Write; use std::sync::Arc; @@ -27,17 +27,16 @@ use risingwave_pb::meta::EventLog; use risingwave_pb::monitor_service::StackTraceResponse; use risingwave_rpc_client::ComputeClientPool; use serde_json::json; +use thiserror_ext::AsReport; use crate::hummock::HummockManagerRef; use crate::manager::event_log::EventLogMangerRef; -use crate::manager::{CatalogManagerRef, ClusterManagerRef, FragmentManagerRef}; +use crate::manager::MetadataManager; pub type DiagnoseCommandRef = Arc; pub struct DiagnoseCommand { - cluster_manager: ClusterManagerRef, - catalog_manager: CatalogManagerRef, - fragment_manager: FragmentManagerRef, + metadata_manager: MetadataManager, hummock_manger: HummockManagerRef, event_log_manager: EventLogMangerRef, prometheus_client: Option, @@ -46,18 +45,14 @@ pub struct DiagnoseCommand { impl DiagnoseCommand { pub fn new( - cluster_manager: ClusterManagerRef, - catalog_manager: CatalogManagerRef, - fragment_manager: FragmentManagerRef, + metadata_manager: MetadataManager, hummock_manger: HummockManagerRef, event_log_manager: EventLogMangerRef, prometheus_client: Option, prometheus_selector: String, ) -> Self { Self { - cluster_manager, - catalog_manager, - fragment_manager, + metadata_manager, hummock_manger, event_log_manager, prometheus_client, @@ -90,49 +85,60 @@ impl DiagnoseCommand { #[cfg_attr(coverage, coverage(off))] async fn write_catalog(&self, s: &mut String) { + match &self.metadata_manager { + MetadataManager::V1(_) => self.write_catalog_v1(s).await, + MetadataManager::V2(_) => self.write_catalog_v2(s).await, + } + } + + #[cfg_attr(coverage, coverage(off))] + async fn write_catalog_v1(&self, s: &mut String) { + let mgr = self.metadata_manager.as_v1_ref(); let _ = writeln!(s, "number of fragment: {}", self.fragment_num().await); let _ = writeln!(s, "number of actor: {}", self.actor_num().await); let _ = writeln!( s, "number of source: {}", - self.catalog_manager.source_count().await + mgr.catalog_manager.source_count().await ); let _ = writeln!( s, "number of table: {}", - self.catalog_manager.table_count().await + mgr.catalog_manager.table_count().await ); let _ = writeln!( s, "number of materialized view: {}", - self.catalog_manager.materialized_view_count().await + mgr.catalog_manager.materialized_view_count().await ); let _ = writeln!( s, "number of sink: {}", - self.catalog_manager.sink_count().await + mgr.catalog_manager.sink_count().await ); let _ = writeln!( s, "number of index: {}", - self.catalog_manager.index_count().await + mgr.catalog_manager.index_count().await ); let _ = writeln!( s, "number of function: {}", - self.catalog_manager.function_count().await + mgr.catalog_manager.function_count().await ); } #[cfg_attr(coverage, coverage(off))] async fn fragment_num(&self) -> usize { - let core = self.fragment_manager.get_fragment_read_guard().await; + let mgr = self.metadata_manager.as_v1_ref(); + let core = mgr.fragment_manager.get_fragment_read_guard().await; core.table_fragments().len() } #[cfg_attr(coverage, coverage(off))] async fn actor_num(&self) -> usize { - let core = self.fragment_manager.get_fragment_read_guard().await; + let mgr = self.metadata_manager.as_v1_ref(); + let core = mgr.fragment_manager.get_fragment_read_guard().await; core.table_fragments() .values() .map(|t| t.actor_status.len()) @@ -140,25 +146,38 @@ impl DiagnoseCommand { } #[cfg_attr(coverage, coverage(off))] - async fn write_worker_nodes(&self, s: &mut String) { - let mut worker_actor_count: HashMap = HashMap::new(); - for f in self - .fragment_manager - .get_fragment_read_guard() - .await - .table_fragments() - .values() - { - for a in f.actor_status.values() { - if let Some(pu) = &a.parallel_unit { - let e = worker_actor_count.entry(pu.worker_node_id).or_insert(0); - *e += 1; - } + async fn write_catalog_v2(&self, s: &mut String) { + let mgr = self.metadata_manager.as_v2_ref(); + let guard = mgr.catalog_controller.get_inner_read_guard().await; + let stat = match guard.stats().await { + Ok(stat) => stat, + Err(err) => { + tracing::warn!(error=?err.as_report(), "failed to get catalog stats"); + return; } - } + }; + let _ = writeln!(s, "number of fragment: {}", stat.streaming_job_num); + let _ = writeln!(s, "number of actor: {}", stat.actor_num); + let _ = writeln!(s, "number of source: {}", stat.source_num); + let _ = writeln!(s, "number of table: {}", stat.table_num); + let _ = writeln!(s, "number of materialized view: {}", stat.mview_num); + let _ = writeln!(s, "number of sink: {}", stat.sink_num); + let _ = writeln!(s, "number of index: {}", stat.index_num); + let _ = writeln!(s, "number of function: {}", stat.function_num); + } + + #[cfg_attr(coverage, coverage(off))] + async fn write_worker_nodes(&self, s: &mut String) { + let Ok(worker_actor_count) = self.metadata_manager.worker_actor_count().await else { + tracing::warn!("failed to get worker actor count"); + return; + }; use comfy_table::{Row, Table}; - let worker_nodes = self.cluster_manager.list_worker_node(None, None).await; + let Ok(worker_nodes) = self.metadata_manager.list_worker_node(None, None).await else { + tracing::warn!("failed to get worker nodes"); + return; + }; let mut table = Table::new(); table.set_header({ let mut row = Row::new(); @@ -636,10 +655,14 @@ impl DiagnoseCommand { #[cfg_attr(coverage, coverage(off))] async fn write_await_tree(&self, s: &mut String) { // Most lines of code are copied from dashboard::handlers::dump_await_tree_all, because the latter cannot be called directly from here. - let worker_nodes = self - .cluster_manager + let Ok(worker_nodes) = self + .metadata_manager .list_worker_node(Some(WorkerType::ComputeNode), None) - .await; + .await + else { + tracing::warn!("failed to get worker nodes"); + return; + }; let mut all = Default::default(); diff --git a/src/meta/src/manager/env.rs b/src/meta/src/manager/env.rs index 3cc909fb28f26..ea26fea83ebcc 100644 --- a/src/meta/src/manager/env.rs +++ b/src/meta/src/manager/env.rs @@ -45,13 +45,13 @@ use crate::MetaResult; #[derive(Clone)] pub struct MetaSrvEnv { /// id generator manager. - id_gen_manager: IdGeneratorManagerRef, + id_gen_manager: Option, /// sql id generator manager. sql_id_gen_manager: Option, /// meta store. - meta_store: MetaStoreRef, + meta_store: Option, /// sql meta store. meta_store_sql: Option, @@ -68,7 +68,7 @@ pub struct MetaSrvEnv { event_log_manager: EventLogMangerRef, /// system param manager. - system_params_manager: SystemParamsManagerRef, + system_params_manager: Option, /// system param controller. system_params_controller: Option, @@ -76,9 +76,6 @@ pub struct MetaSrvEnv { /// Unique identifier of the cluster. cluster_id: ClusterId, - /// Whether the cluster is launched for the first time. - cluster_first_launch: bool, - /// Client to connector node. `None` if endpoint unspecified or unable to connect. connector_client: Option, @@ -94,11 +91,14 @@ pub struct MetaOpts { /// Whether to enable the recovery of the cluster. If disabled, the meta service will exit on /// abnormal cases. pub enable_recovery: bool, - /// Whether to enable the scale-in feature when compute-node is removed. - pub enable_scale_in_when_recovery: bool, - /// Whether to enable the auto-scaling feature when compute-node is joined. - /// The semantics of this configuration will be expanded in the future to control the automatic scaling of the entire cluster. - pub enable_automatic_parallelism_control: bool, + /// Whether to disable the auto-scaling feature. + pub disable_automatic_parallelism_control: bool, + /// The number of streaming jobs per scaling operation. + pub parallelism_control_batch_size: usize, + /// The period of parallelism control trigger. + pub parallelism_control_trigger_period_sec: u64, + /// The first delay of parallelism control. + pub parallelism_control_trigger_first_delay_sec: u64, /// The maximum number of barriers in-flight in the compute nodes. pub in_flight_barrier_nums: usize, /// After specified seconds of idle (no mview or flush), the process will be exited. @@ -117,6 +117,7 @@ pub struct MetaOpts { pub vacuum_spin_interval_ms: u64, /// Interval of hummock version checkpoint. pub hummock_version_checkpoint_interval_sec: u64, + pub enable_hummock_data_archive: bool, /// The minimum delta log number a new checkpoint should compact, otherwise the checkpoint /// attempt is rejected. Greater value reduces object store IO, meanwhile it results in /// more loss of in memory `HummockVersionCheckpoint::stale_objects` state when meta node is @@ -212,6 +213,12 @@ pub struct MetaOpts { /// The maximum memory usage in bytes for the tracing collector embedded /// in the meta node. pub cached_traces_memory_limit_bytes: usize, + + /// l0 picker whether to select trivial move task + pub enable_trivial_move: bool, + + /// l0 multi level picker whether to check the overlap accuracy between sub levels + pub enable_check_task_level_overlap: bool, } impl MetaOpts { @@ -219,8 +226,10 @@ impl MetaOpts { pub fn test(enable_recovery: bool) -> Self { Self { enable_recovery, - enable_scale_in_when_recovery: false, - enable_automatic_parallelism_control: false, + disable_automatic_parallelism_control: false, + parallelism_control_batch_size: 1, + parallelism_control_trigger_period_sec: 10, + parallelism_control_trigger_first_delay_sec: 30, in_flight_barrier_nums: 40, max_idle_ms: 0, compaction_deterministic_test: false, @@ -228,6 +237,7 @@ impl MetaOpts { vacuum_interval_sec: 30, vacuum_spin_interval_ms: 0, hummock_version_checkpoint_interval_sec: 30, + enable_hummock_data_archive: false, min_delta_log_num_for_hummock_version_checkpoint: 1, min_sst_retention_time_sec: 3600 * 24 * 7, full_gc_interval_sec: 3600 * 24 * 7, @@ -262,6 +272,8 @@ impl MetaOpts { advertise_addr: "".to_string(), cached_traces_num: 1, cached_traces_memory_limit_bytes: usize::MAX, + enable_trivial_move: true, + enable_check_task_level_overlap: true, } } } @@ -270,38 +282,50 @@ impl MetaSrvEnv { pub async fn new( opts: MetaOpts, init_system_params: SystemParams, - meta_store: MetaStoreRef, + meta_store: Option, meta_store_sql: Option, ) -> MetaResult { - // change to sync after refactor `IdGeneratorManager::new` sync. - let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await); - let stream_client_pool = Arc::new(StreamClientPool::default()); - let notification_manager = Arc::new(NotificationManager::new(meta_store.clone()).await); + let notification_manager = + Arc::new(NotificationManager::new(meta_store.clone(), meta_store_sql.clone()).await); let idle_manager = Arc::new(IdleManager::new(opts.max_idle_ms)); - let (mut cluster_id, cluster_first_launch) = - if let Some(id) = ClusterId::from_meta_store(&meta_store).await? { - (id, false) - } else { - (ClusterId::new(), true) - }; - let system_params_manager = Arc::new( - SystemParamsManager::new( - meta_store.clone(), - notification_manager.clone(), - init_system_params.clone(), - cluster_first_launch, - ) - .await?, - ); - // TODO: remove `cluster_first_launch` and check equality of cluster id stored in hummock to - // make sure the data dir of hummock is not used by another cluster. + let stream_client_pool = Arc::new(StreamClientPool::default()); + + let (id_gen_manager, mut cluster_id, system_params_manager) = match meta_store.clone() { + Some(meta_store) => { + // change to sync after refactor `IdGeneratorManager::new` sync. + let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await); + let (cluster_id, cluster_first_launch) = + if let Some(id) = ClusterId::from_meta_store(&meta_store).await? { + (id, false) + } else { + (ClusterId::new(), true) + }; + let system_params_manager = Arc::new( + SystemParamsManager::new( + meta_store.clone(), + notification_manager.clone(), + init_system_params.clone(), + cluster_first_launch, + ) + .await?, + ); + ( + Some(id_gen_manager), + Some(cluster_id), + Some(system_params_manager), + ) + } + None => (None, None, None), + }; let system_params_controller = match &meta_store_sql { Some(store) => { - cluster_id = Cluster::find() - .one(&store.conn) - .await? - .map(|c| c.cluster_id.to_string().into()) - .unwrap(); + cluster_id = Some( + Cluster::find() + .one(&store.conn) + .await? + .map(|c| c.cluster_id.to_string().into()) + .unwrap(), + ); Some(Arc::new( SystemParamsController::new( store.clone(), @@ -322,7 +346,6 @@ impl MetaSrvEnv { let hummock_seq = meta_store_sql .clone() .map(|m| Arc::new(SequenceGenerator::new(m.conn))); - let sql_id_gen_manager = if let Some(store) = &meta_store_sql { Some(Arc::new(SqlIdGeneratorManager::new(&store.conn).await?)) } else { @@ -340,8 +363,7 @@ impl MetaSrvEnv { event_log_manager, system_params_manager, system_params_controller, - cluster_id, - cluster_first_launch, + cluster_id: cluster_id.unwrap(), connector_client, opts: opts.into(), hummock_seq, @@ -349,23 +371,23 @@ impl MetaSrvEnv { } pub fn meta_store_ref(&self) -> MetaStoreRef { - self.meta_store.clone() + self.meta_store.clone().unwrap() } - pub fn meta_store(&self) -> &MetaStoreRef { - &self.meta_store + pub fn meta_store_checked(&self) -> &MetaStoreRef { + self.meta_store.as_ref().unwrap() } - pub fn sql_meta_store(&self) -> Option { - self.meta_store_sql.clone() + pub fn meta_store(&self) -> Option<&MetaStoreRef> { + self.meta_store.as_ref() } - pub fn id_gen_manager_ref(&self) -> IdGeneratorManagerRef { - self.id_gen_manager.clone() + pub fn sql_meta_store(&self) -> Option { + self.meta_store_sql.clone() } pub fn id_gen_manager(&self) -> &IdGeneratorManager { - self.id_gen_manager.deref() + self.id_gen_manager.as_ref().unwrap() } pub fn sql_id_gen_manager_ref(&self) -> Option { @@ -392,15 +414,19 @@ impl MetaSrvEnv { if let Some(system_ctl) = &self.system_params_controller { return system_ctl.get_params().await; } - self.system_params_manager.get_params().await + self.system_params_manager + .as_ref() + .unwrap() + .get_params() + .await } - pub fn system_params_manager_ref(&self) -> SystemParamsManagerRef { + pub fn system_params_manager_ref(&self) -> Option { self.system_params_manager.clone() } - pub fn system_params_manager(&self) -> &SystemParamsManager { - self.system_params_manager.deref() + pub fn system_params_manager(&self) -> Option<&SystemParamsManagerRef> { + self.system_params_manager.as_ref() } pub fn system_params_controller_ref(&self) -> Option { @@ -423,10 +449,6 @@ impl MetaSrvEnv { &self.cluster_id } - pub fn cluster_first_launch(&self) -> bool { - self.cluster_first_launch - } - pub fn connector_client(&self) -> Option { self.connector_client.clone() } @@ -446,18 +468,19 @@ impl MetaSrvEnv { pub async fn for_test_opts(opts: Arc) -> Self { use crate::manager::event_log::EventLogManger; - // change to sync after refactor `IdGeneratorManager::new` sync. let meta_store = MemStore::default().into_ref(); #[cfg(madsim)] let meta_store_sql: Option = None; #[cfg(not(madsim))] let meta_store_sql = Some(SqlMetaStore::for_test().await); - let id_gen_manager = Arc::new(IdGeneratorManager::new(meta_store.clone()).await); - let notification_manager = Arc::new(NotificationManager::new(meta_store.clone()).await); + let id_gen_manager = Some(Arc::new(IdGeneratorManager::new(meta_store.clone()).await)); + let notification_manager = Arc::new( + NotificationManager::new(Some(meta_store.clone()), meta_store_sql.clone()).await, + ); let stream_client_pool = Arc::new(StreamClientPool::default()); let idle_manager = Arc::new(IdleManager::disabled()); - let (cluster_id, cluster_first_launch) = (ClusterId::new(), true); + let cluster_id = ClusterId::new(); let system_params_manager = Arc::new( SystemParamsManager::new( meta_store.clone(), @@ -497,16 +520,15 @@ impl MetaSrvEnv { Self { id_gen_manager, sql_id_gen_manager, - meta_store, + meta_store: Some(meta_store), meta_store_sql, notification_manager, stream_client_pool, idle_manager, event_log_manager, - system_params_manager, + system_params_manager: Some(system_params_manager), system_params_controller, cluster_id, - cluster_first_launch, connector_client: None, opts, hummock_seq, diff --git a/src/meta/src/manager/id.rs b/src/meta/src/manager/id.rs index f8373e160961e..377589886ec6c 100644 --- a/src/meta/src/manager/id.rs +++ b/src/meta/src/manager/id.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use risingwave_common::catalog::{NON_RESERVED_SYS_CATALOG_ID, NON_RESERVED_USER_ID}; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; +use thiserror_ext::AsReport; use tokio::sync::RwLock; use crate::manager::cluster::META_NODE_ID; @@ -57,7 +58,7 @@ impl StoredIdGenerator { let current_id = match res { Ok(value) => memcomparable::from_slice(&value).unwrap(), Err(MetaStoreError::ItemNotFound(_)) => start.unwrap_or(0), - Err(e) => panic!("{:?}", e), + Err(e) => panic!("{}", e.as_report()), }; let next_allocate_id = current_id + ID_PREALLOCATE_INTERVAL; @@ -69,7 +70,7 @@ impl StoredIdGenerator { ) .await { - panic!("{:?}", err) + panic!("{}", err.as_report()); } StoredIdGenerator { diff --git a/src/meta/src/manager/metadata.rs b/src/meta/src/manager/metadata.rs index 450a920c379d0..fd9b51fa8ba09 100644 --- a/src/meta/src/manager/metadata.rs +++ b/src/meta/src/manager/metadata.rs @@ -12,23 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{HashMap, HashSet}; +use std::collections::{BTreeMap, HashMap, HashSet}; use risingwave_common::catalog::{TableId, TableOption}; use risingwave_meta_model_v2::SourceId; -use risingwave_pb::catalog::PbSource; +use risingwave_pb::catalog::{PbSource, PbTable}; use risingwave_pb::common::worker_node::{PbResource, State}; -use risingwave_pb::common::{HostAddress, PbWorkerNode, PbWorkerType, WorkerType}; +use risingwave_pb::common::{HostAddress, PbWorkerNode, PbWorkerType, WorkerNode, WorkerType}; use risingwave_pb::meta::add_worker_node_request::Property as AddNodeProperty; -use risingwave_pb::meta::table_fragments::Fragment; -use risingwave_pb::stream_plan::PbStreamActor; +use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, PbFragment}; +use risingwave_pb::stream_plan::{PbDispatchStrategy, PbStreamActor, StreamActor}; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver}; +use tracing::warn; +use crate::barrier::Reschedule; use crate::controller::catalog::CatalogControllerRef; use crate::controller::cluster::{ClusterControllerRef, WorkerExtraInfo}; use crate::manager::{ - CatalogManagerRef, ClusterManagerRef, FragmentManagerRef, StreamingClusterInfo, WorkerId, + CatalogManagerRef, ClusterManagerRef, FragmentManagerRef, LocalNotification, + StreamingClusterInfo, WorkerId, }; -use crate::model::{ActorId, FragmentId, MetadataModel, TableFragments}; +use crate::model::{ActorId, FragmentId, MetadataModel, TableFragments, TableParallelism}; +use crate::stream::SplitAssignment; use crate::MetaResult; #[derive(Clone)] @@ -50,6 +55,127 @@ pub struct MetadataManagerV2 { pub catalog_controller: CatalogControllerRef, } +#[derive(Debug)] +pub(crate) enum ActiveStreamingWorkerChange { + Add(WorkerNode), + Remove(WorkerNode), + Update(WorkerNode), +} + +pub struct ActiveStreamingWorkerNodes { + worker_nodes: HashMap, + rx: UnboundedReceiver, +} + +impl ActiveStreamingWorkerNodes { + pub(crate) fn uninitialized() -> Self { + Self { + worker_nodes: Default::default(), + rx: unbounded_channel().1, + } + } + + /// Return an uninitialized one as a place holder for future initialized + pub(crate) async fn new_snapshot(meta_manager: MetadataManager) -> MetaResult { + let (nodes, rx) = meta_manager + .subscribe_active_streaming_compute_nodes() + .await?; + Ok(Self { + worker_nodes: nodes.into_iter().map(|node| (node.id, node)).collect(), + rx, + }) + } + + pub(crate) fn current(&self) -> &HashMap { + &self.worker_nodes + } + + pub(crate) async fn changed(&mut self) -> ActiveStreamingWorkerChange { + let ret = loop { + let notification = self + .rx + .recv() + .await + .expect("notification stopped or uninitialized"); + match notification { + LocalNotification::WorkerNodeDeleted(worker) => { + let is_streaming_compute_node = worker.r#type == WorkerType::ComputeNode as i32 + && worker.property.as_ref().unwrap().is_streaming; + let Some(prev_worker) = self.worker_nodes.remove(&worker.id) else { + if is_streaming_compute_node { + warn!( + ?worker, + "notify to delete an non-existing streaming compute worker" + ); + } + continue; + }; + if !is_streaming_compute_node { + warn!( + ?worker, + ?prev_worker, + "deleted worker has a different recent type" + ); + } + if worker.state == State::Starting as i32 { + warn!( + id = worker.id, + host = ?worker.host, + state = worker.state, + "a starting streaming worker is deleted" + ); + } + break ActiveStreamingWorkerChange::Remove(prev_worker); + } + LocalNotification::WorkerNodeActivated(worker) => { + if worker.r#type != WorkerType::ComputeNode as i32 + || !worker.property.as_ref().unwrap().is_streaming + { + if let Some(prev_worker) = self.worker_nodes.remove(&worker.id) { + warn!( + ?worker, + ?prev_worker, + "the type of a streaming worker is changed" + ); + break ActiveStreamingWorkerChange::Remove(prev_worker); + } else { + continue; + } + } + assert_eq!( + worker.state, + State::Running as i32, + "not started worker added: {:?}", + worker + ); + if let Some(prev_worker) = self.worker_nodes.insert(worker.id, worker.clone()) { + assert_eq!(prev_worker.host, worker.host); + assert_eq!(prev_worker.r#type, worker.r#type); + warn!( + ?prev_worker, + ?worker, + eq = prev_worker == worker, + "notify to update an existing active worker" + ); + if prev_worker == worker { + continue; + } else { + break ActiveStreamingWorkerChange::Update(worker); + } + } else { + break ActiveStreamingWorkerChange::Add(worker); + } + } + _ => { + continue; + } + } + }; + + ret + } +} + impl MetadataManager { pub fn new_v1( cluster_manager: ClusterManagerRef, @@ -73,6 +199,20 @@ impl MetadataManager { }) } + pub fn as_v1_ref(&self) -> &MetadataManagerV1 { + match self { + MetadataManager::V1(mgr) => mgr, + MetadataManager::V2(_) => panic!("expect v1, found v2"), + } + } + + pub fn as_v2_ref(&self) -> &MetadataManagerV2 { + match self { + MetadataManager::V1(_) => panic!("expect v2, found v1"), + MetadataManager::V2(mgr) => mgr, + } + } + pub async fn get_worker_by_id(&self, worker_id: WorkerId) -> MetaResult> { match &self { MetadataManager::V1(mgr) => Ok(mgr @@ -155,6 +295,22 @@ impl MetadataManager { } } + pub async fn subscribe_active_streaming_compute_nodes( + &self, + ) -> MetaResult<(Vec, UnboundedReceiver)> { + match self { + MetadataManager::V1(mgr) => Ok(mgr + .cluster_manager + .subscribe_active_streaming_compute_nodes() + .await), + MetadataManager::V2(mgr) => { + mgr.cluster_controller + .subscribe_active_streaming_compute_nodes() + .await + } + } + } + pub async fn list_active_streaming_compute_nodes(&self) -> MetaResult> { match self { MetadataManager::V1(mgr) => Ok(mgr @@ -174,6 +330,69 @@ impl MetadataManager { } } + pub async fn pre_apply_reschedules( + &self, + created_actors: HashMap>, + ) -> HashMap> { + match self { + MetadataManager::V1(mgr) => { + mgr.fragment_manager + .pre_apply_reschedules(created_actors) + .await + } + + // V2 doesn't need to pre apply reschedules. + MetadataManager::V2(_) => HashMap::new(), + } + } + + pub async fn post_apply_reschedules( + &self, + reschedules: HashMap, + table_parallelism_assignment: HashMap, + ) -> MetaResult<()> { + match self { + MetadataManager::V1(mgr) => { + mgr.fragment_manager + .post_apply_reschedules(reschedules, table_parallelism_assignment) + .await + } + MetadataManager::V2(mgr) => { + // temp convert u32 to i32 + let reschedules = reschedules.into_iter().map(|(k, v)| (k as _, v)).collect(); + + mgr.catalog_controller + .post_apply_reschedules(reschedules, table_parallelism_assignment) + .await + } + } + } + + pub async fn running_fragment_parallelisms( + &self, + id_filter: Option>, + ) -> MetaResult> { + match self { + MetadataManager::V1(mgr) => Ok(mgr + .fragment_manager + .running_fragment_parallelisms(id_filter) + .await + .into_iter() + .map(|(k, v)| (k as FragmentId, v)) + .collect()), + MetadataManager::V2(mgr) => { + let id_filter = id_filter.map(|ids| ids.into_iter().map(|id| id as _).collect()); + Ok(mgr + .catalog_controller + .running_fragment_parallelisms(id_filter) + .await? + .into_iter() + .map(|(k, v)| (k as FragmentId, v)) + .collect()) + } + } + } + /// Get and filter the "**root**" fragments of the specified relations. /// The root fragment is the bottom-most fragment of its fragment graph, and can be a `MView` or a `Source`. /// @@ -259,6 +478,59 @@ impl MetadataManager { } } + pub async fn get_table_catalog_by_ids(&self, ids: Vec) -> MetaResult> { + match &self { + MetadataManager::V1(mgr) => Ok(mgr.catalog_manager.get_tables(&ids).await), + MetadataManager::V2(mgr) => { + mgr.catalog_controller + .get_table_by_ids(ids.into_iter().map(|id| id as _).collect()) + .await + } + } + } + + pub async fn get_downstream_chain_fragments( + &self, + job_id: u32, + ) -> MetaResult> { + match &self { + MetadataManager::V1(mgr) => { + mgr.fragment_manager + .get_downstream_fragments(job_id.into()) + .await + } + MetadataManager::V2(mgr) => { + mgr.catalog_controller + .get_downstream_chain_fragments(job_id as _) + .await + } + } + } + + pub async fn get_worker_actor_ids( + &self, + job_ids: HashSet, + ) -> MetaResult>> { + match &self { + MetadataManager::V1(mgr) => mgr.fragment_manager.table_node_actors(&job_ids).await, + MetadataManager::V2(mgr) => { + let worker_actors = mgr + .catalog_controller + .get_worker_actor_ids(job_ids.into_iter().map(|id| id.table_id as _).collect()) + .await?; + Ok(worker_actors + .into_iter() + .map(|(id, actors)| { + ( + id as WorkerId, + actors.into_iter().map(|id| id as ActorId).collect(), + ) + }) + .collect()) + } + } + } + pub async fn get_job_id_to_internal_table_ids_mapping(&self) -> Option)>> { match &self { MetadataManager::V1(mgr) => mgr @@ -365,6 +637,30 @@ impl MetadataManager { } } + pub async fn worker_actor_count(&self) -> MetaResult> { + match &self { + MetadataManager::V1(mgr) => Ok(mgr.fragment_manager.node_actor_count().await), + MetadataManager::V2(mgr) => { + let actor_cnt = mgr.catalog_controller.worker_actor_count().await?; + Ok(actor_cnt + .into_iter() + .map(|(id, cnt)| (id as WorkerId, cnt)) + .collect()) + } + } + } + + pub async fn count_streaming_job(&self) -> MetaResult { + match self { + MetadataManager::V1(mgr) => Ok(mgr.fragment_manager.count_streaming_job().await), + MetadataManager::V2(mgr) => mgr + .catalog_controller + .list_streaming_job_states() + .await + .map(|x| x.len()), + } + } + pub async fn drop_streaming_job_by_ids(&self, table_ids: &HashSet) -> MetaResult<()> { match self { MetadataManager::V1(mgr) => { @@ -426,4 +722,22 @@ impl MetadataManager { } } } + + pub async fn update_actor_splits_by_split_assignment( + &self, + split_assignment: &SplitAssignment, + ) -> MetaResult<()> { + match self { + MetadataManager::V1(mgr) => { + mgr.fragment_manager + .update_actor_splits_by_split_assignment(split_assignment) + .await + } + MetadataManager::V2(mgr) => { + mgr.catalog_controller + .update_actor_splits(split_assignment) + .await + } + } + } } diff --git a/src/meta/src/manager/mod.rs b/src/meta/src/manager/mod.rs index 4ab91c388792d..24c9810084259 100644 --- a/src/meta/src/manager/mod.rs +++ b/src/meta/src/manager/mod.rs @@ -21,6 +21,7 @@ mod id; mod idle; mod metadata; mod notification; +mod notification_version; pub mod sink_coordination; mod streaming_job; mod system_param; diff --git a/src/meta/src/manager/notification.rs b/src/meta/src/manager/notification.rs index 2319ce35d5c8c..166f35655aa3c 100644 --- a/src/meta/src/manager/notification.rs +++ b/src/meta/src/manager/notification.rs @@ -23,12 +23,15 @@ use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::meta::{ MetaSnapshot, Relation, RelationGroup, SubscribeResponse, SubscribeType, }; +use thiserror_ext::AsReport; use tokio::sync::mpsc::{self, UnboundedSender}; use tokio::sync::Mutex; use tonic::Status; +use crate::controller::SqlMetaStore; use crate::manager::cluster::WorkerKey; -use crate::model::{FragmentId, NotificationVersion as Version}; +use crate::manager::notification_version::NotificationVersionGenerator; +use crate::model::FragmentId; use crate::storage::MetaStoreRef; pub type MessageStatus = Status; @@ -76,17 +79,22 @@ pub struct NotificationManager { core: Arc>, /// Sender used to add a notification into the waiting queue. task_tx: UnboundedSender, - /// The current notification version. - current_version: Mutex, - meta_store: MetaStoreRef, + /// The current notification version generator. + version_generator: Mutex, } impl NotificationManager { - pub async fn new(meta_store: MetaStoreRef) -> Self { + pub async fn new( + meta_store: Option, + meta_store_sql: Option, + ) -> Self { // notification waiting queue. let (task_tx, mut task_rx) = mpsc::unbounded_channel::(); let core = Arc::new(Mutex::new(NotificationManagerCore::new())); let core_clone = core.clone(); + let version_generator = NotificationVersionGenerator::new(meta_store, meta_store_sql) + .await + .unwrap(); tokio::spawn(async move { while let Some(task) = task_rx.recv().await { @@ -104,8 +112,7 @@ impl NotificationManager { Self { core: core_clone, task_tx, - current_version: Mutex::new(Version::new(&meta_store).await), - meta_store, + version_generator: Mutex::new(version_generator), } } @@ -139,9 +146,9 @@ impl NotificationManager { operation: Operation, info: Info, ) -> NotificationVersion { - let mut version_guard = self.current_version.lock().await; - version_guard.increase_version(&self.meta_store).await; - let version = version_guard.version(); + let mut version_guard = self.version_generator.lock().await; + version_guard.increase_version().await; + let version = version_guard.current_version(); self.notify(target, operation, info, Some(version)); version } @@ -251,6 +258,10 @@ impl NotificationManager { self.notify_without_version(SubscribeType::Hummock.into(), operation, info) } + pub fn notify_compactor_without_version(&self, operation: Operation, info: Info) { + self.notify_without_version(SubscribeType::Compactor.into(), operation, info) + } + #[cfg(any(test, feature = "test"))] pub fn notify_hummock_with_version( &self, @@ -265,7 +276,7 @@ impl NotificationManager { let mut core_guard = self.core.lock().await; core_guard.local_senders.retain(|sender| { if let Err(err) = sender.send(notification.clone()) { - tracing::warn!("Failed to notify local subscriber. {}", err); + tracing::warn!(error = %err.as_report(), "Failed to notify local subscriber"); return false; } true @@ -319,8 +330,8 @@ impl NotificationManager { } pub async fn current_version(&self) -> NotificationVersion { - let version_guard = self.current_version.lock().await; - version_guard.version() + let version_guard = self.version_generator.lock().await; + version_guard.current_version() } } @@ -410,7 +421,7 @@ mod tests { #[tokio::test] async fn test_multiple_subscribers_one_worker() { - let mgr = NotificationManager::new(MemStore::new().into_ref()).await; + let mgr = NotificationManager::new(Some(MemStore::new().into_ref()), None).await; let worker_key1 = WorkerKey(HostAddress { host: "a".to_string(), port: 1, diff --git a/src/meta/src/manager/notification_version.rs b/src/meta/src/manager/notification_version.rs new file mode 100644 index 0000000000000..18c8c7cc9d20f --- /dev/null +++ b/src/meta/src/manager/notification_version.rs @@ -0,0 +1,85 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_meta_model_v2::catalog_version; +use risingwave_meta_model_v2::catalog_version::VersionCategory; +use risingwave_meta_model_v2::prelude::CatalogVersion; +use sea_orm::ActiveValue::Set; +use sea_orm::{ActiveModelTrait, DatabaseConnection, EntityTrait, TransactionTrait}; + +use crate::controller::SqlMetaStore; +use crate::model::NotificationVersion as NotificationModelV1; +use crate::storage::MetaStoreRef; +use crate::MetaResult; + +pub enum NotificationVersionGenerator { + KvGenerator(NotificationModelV1, MetaStoreRef), + SqlGenerator(u64, DatabaseConnection), +} + +// TODO: add pre-allocation if necessary +impl NotificationVersionGenerator { + pub async fn new( + meta_store: Option, + meta_store_sql: Option, + ) -> MetaResult { + if let Some(sql) = meta_store_sql { + let txn = sql.conn.begin().await?; + let model = CatalogVersion::find_by_id(VersionCategory::Notification) + .one(&txn) + .await?; + let current_version = model.as_ref().map(|m| m.version).unwrap_or(1) as u64; + if model.is_none() { + CatalogVersion::insert(catalog_version::ActiveModel { + name: Set(VersionCategory::Notification), + version: Set(1), + }) + .exec(&txn) + .await?; + txn.commit().await?; + } + + Ok(Self::SqlGenerator(current_version, sql.conn)) + } else { + let meta_store = meta_store.unwrap(); + let current_version = NotificationModelV1::new(&meta_store).await; + Ok(Self::KvGenerator(current_version, meta_store)) + } + } + + pub fn current_version(&self) -> u64 { + match self { + NotificationVersionGenerator::KvGenerator(v, _) => v.version(), + NotificationVersionGenerator::SqlGenerator(v, _) => *v, + } + } + + pub async fn increase_version(&mut self) { + match self { + NotificationVersionGenerator::KvGenerator(v, meta_store) => { + v.increase_version(meta_store).await + } + NotificationVersionGenerator::SqlGenerator(v, conn) => { + catalog_version::ActiveModel { + name: Set(VersionCategory::Notification), + version: Set((*v + 1) as i64), + } + .update(conn) + .await + .unwrap(); + *v += 1; + } + } + } +} diff --git a/src/meta/src/manager/sink_coordination/coordinator_worker.rs b/src/meta/src/manager/sink_coordination/coordinator_worker.rs index 24977acc57607..e1c096aa3cf98 100644 --- a/src/meta/src/manager/sink_coordination/coordinator_worker.rs +++ b/src/meta/src/manager/sink_coordination/coordinator_worker.rs @@ -30,6 +30,7 @@ use risingwave_pb::connector_service::coordinate_response::{ use risingwave_pb::connector_service::{ coordinate_request, coordinate_response, CoordinateRequest, CoordinateResponse, SinkMetadata, }; +use thiserror_ext::AsReport; use tokio::sync::mpsc::UnboundedReceiver; use tonic::Status; use tracing::{error, warn}; @@ -62,8 +63,9 @@ impl CoordinatorWorker { Ok(sink) => sink, Err(e) => { error!( - "unable to build sink with param {:?}: {:?}", - first_writer_request.param, e + error = %e.as_report(), + "unable to build sink with param {:?}", + first_writer_request.param ); send_await_with_err_check!( first_writer_request.response_tx, @@ -77,8 +79,9 @@ impl CoordinatorWorker { Ok(coordinator) => coordinator, Err(e) => { error!( - "unable to build coordinator with param {:?}: {:?}", - first_writer_request.param, e + error = %e.as_report(), + "unable to build coordinator with param {:?}", + first_writer_request.param ); send_await_with_err_check!( first_writer_request.response_tx, @@ -107,7 +110,7 @@ impl CoordinatorWorker { .wait_for_writers(first_writer_request.vnode_bitmap) .await { - error!("failed to wait for all writers: {:?}", e); + error!(error = %e.as_report(), "failed to wait for all writers"); worker .send_to_all_sink_writers(|| { Err(Status::cancelled("failed to wait for all writers")) @@ -149,10 +152,9 @@ impl CoordinatorWorker { Either::Right((Some(Ok(None)), _)) => Err(anyhow!( "one sink writer stream reaches the end before initialize" )), - Either::Right((Some(Err(e)), _)) => Err(anyhow!( - "unable to poll from one sink writer stream: {:?}", - e - )), + Either::Right((Some(Err(e)), _)) => { + Err(anyhow!(e).context("unable to poll from one sink writer stream")) + } Either::Right((None, _)) => unreachable!("request_streams must not be empty"), } } @@ -265,10 +267,8 @@ impl CoordinatorWorker { )); } Err(e) => { - return Err(anyhow!( - "failed to poll from one of the writer request streams: {:?}", - e - )); + return Err(anyhow!(e) + .context("failed to poll from one of the writer request streams")); } }, Either::Right((None, _)) => { @@ -285,17 +285,16 @@ impl CoordinatorWorker { async fn start_coordination(&mut self, mut coordinator: impl SinkCommitCoordinator) { let result: Result<(), ()> = try { coordinator.init().await.map_err(|e| { - error!("failed to initialize coordinator: {:?}", e); + error!(error = %e.as_report(), "failed to initialize coordinator"); })?; loop { let (epoch, metadata_list) = self.collect_all_metadata().await.map_err(|e| { - error!("failed to collect all metadata: {:?}", e); + error!(error = %e.as_report(), "failed to collect all metadata"); })?; // TODO: measure commit time - coordinator - .commit(epoch, metadata_list) - .await - .map_err(|e| error!("failed to commit metadata of epoch {}: {:?}", epoch, e))?; + coordinator.commit(epoch, metadata_list).await.map_err( + |e| error!(epoch, error = %e.as_report(), "failed to commit metadata of epoch"), + )?; self.send_to_all_sink_writers(|| { Ok(CoordinateResponse { diff --git a/src/meta/src/manager/sink_coordination/manager.rs b/src/meta/src/manager/sink_coordination/manager.rs index 34b4073916e6c..d174b8aca7c59 100644 --- a/src/meta/src/manager/sink_coordination/manager.rs +++ b/src/meta/src/manager/sink_coordination/manager.rs @@ -20,11 +20,12 @@ use futures::future::{select, BoxFuture, Either}; use futures::stream::FuturesUnordered; use futures::{FutureExt, Stream, StreamExt, TryStreamExt}; use risingwave_common::buffer::Bitmap; -use risingwave_common::util::pending_on_none; use risingwave_connector::sink::catalog::SinkId; use risingwave_connector::sink::SinkParam; use risingwave_pb::connector_service::coordinate_request::Msg; use risingwave_pb::connector_service::{coordinate_request, CoordinateRequest, CoordinateResponse}; +use rw_futures_util::pending_on_none; +use thiserror_ext::AsReport; use tokio::sync::mpsc; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tokio::sync::oneshot::{channel, Receiver, Sender}; @@ -292,14 +293,15 @@ impl ManagerWorker { match join_result { Ok(()) => { info!( - "sink coordinator of {} has gracefully finished", - sink_id.sink_id + id = sink_id.sink_id, + "sink coordinator has gracefully finished", ); } Err(err) => { error!( - "sink coordinator of {} finished with error {:?}", - sink_id.sink_id, err + id = sink_id.sink_id, + error = %err.as_report(), + "sink coordinator finished with error", ); } } diff --git a/src/meta/src/manager/streaming_job.rs b/src/meta/src/manager/streaming_job.rs index 54dfdb9a22abb..2f7a68ed081f3 100644 --- a/src/meta/src/manager/streaming_job.rs +++ b/src/meta/src/manager/streaming_job.rs @@ -12,12 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; - use risingwave_common::catalog::TableVersionId; use risingwave_common::current_cluster_version; use risingwave_common::util::epoch::Epoch; -use risingwave_pb::catalog::{CreateType, Index, PbSource, Sink, Table}; +use risingwave_pb::catalog::{CreateType, Index, PbSource, Sink, Subscription, Table}; use risingwave_pb::ddl_service::TableJobType; use strum::EnumDiscriminants; @@ -28,6 +26,7 @@ use crate::model::FragmentId; #[derive(Debug, Clone, EnumDiscriminants)] pub enum StreamingJob { MaterializedView(Table), + Subscription(Subscription), Sink(Sink, Option<(Table, Option)>), Table(Option, Table, TableJobType), Index(Index, Table), @@ -38,6 +37,7 @@ pub enum StreamingJob { pub enum DdlType { MaterializedView, Sink, + Subscription, Table(TableJobType), Index, Source, @@ -51,6 +51,7 @@ impl From<&StreamingJob> for DdlType { StreamingJob::Table(_, _, ty) => DdlType::Table(*ty), StreamingJob::Index(_, _) => DdlType::Index, StreamingJob::Source(_) => DdlType::Source, + StreamingJob::Subscription(_) => DdlType::Subscription, } } } @@ -91,6 +92,10 @@ impl StreamingJob { source.created_at_epoch = created_at_epoch; source.created_at_cluster_version = created_at_cluster_version; } + StreamingJob::Subscription(subscription) => { + subscription.created_at_epoch = created_at_epoch; + subscription.created_at_cluster_version = created_at_cluster_version; + } } } @@ -123,6 +128,10 @@ impl StreamingJob { source.initialized_at_epoch = initialized_at_epoch; source.initialized_at_cluster_version = initialized_at_cluster_version; } + StreamingJob::Subscription(subscription) => { + subscription.initialized_at_epoch = initialized_at_epoch; + subscription.initialized_at_cluster_version = initialized_at_cluster_version; + } } } } @@ -141,6 +150,9 @@ impl StreamingJob { StreamingJob::Source(src) => { src.id = id; } + StreamingJob::Subscription(subscription) => { + subscription.id = id; + } } } @@ -150,7 +162,7 @@ impl StreamingJob { Self::MaterializedView(table) | Self::Index(_, table) | Self::Table(_, table, ..) => { table.fragment_id = id; } - Self::Sink(_, _) | Self::Source(_) => {} + Self::Sink(_, _) | Self::Source(_) | Self::Subscription(_) => {} } } @@ -160,7 +172,10 @@ impl StreamingJob { Self::Table(_, table, ..) => { table.dml_fragment_id = id; } - Self::MaterializedView(_) | Self::Index(_, _) | Self::Sink(_, _) => {} + Self::MaterializedView(_) + | Self::Index(_, _) + | Self::Sink(_, _) + | Self::Subscription(_) => {} Self::Source(_) => {} } } @@ -172,6 +187,7 @@ impl StreamingJob { Self::Table(_, table, ..) => table.id, Self::Index(index, _) => index.id, Self::Source(source) => source.id, + Self::Subscription(subscription) => subscription.id, } } @@ -182,6 +198,7 @@ impl StreamingJob { Self::Table(_, table, ..) => Some(table.id), Self::Index(_, table) => Some(table.id), Self::Source(_) => None, + Self::Subscription(_) => None, } } @@ -191,7 +208,7 @@ impl StreamingJob { Self::MaterializedView(table) | Self::Index(_, table) | Self::Table(_, table, ..) => { Some(table) } - Self::Sink(_, _) | Self::Source(_) => None, + Self::Sink(_, _) | Self::Source(_) | Self::Subscription(_) => None, } } @@ -202,6 +219,7 @@ impl StreamingJob { Self::Table(_, table, ..) => table.schema_id, Self::Index(index, _) => index.schema_id, Self::Source(source) => source.schema_id, + Self::Subscription(subscription) => subscription.schema_id, } } @@ -212,6 +230,7 @@ impl StreamingJob { Self::Table(_, table, ..) => table.database_id, Self::Index(index, _) => index.database_id, Self::Source(source) => source.database_id, + Self::Subscription(subscription) => subscription.database_id, } } @@ -222,6 +241,7 @@ impl StreamingJob { Self::Table(_, table, ..) => table.name.clone(), Self::Index(index, _) => index.name.clone(), Self::Source(source) => source.name.clone(), + Self::Subscription(subscription) => subscription.name.clone(), } } @@ -232,6 +252,7 @@ impl StreamingJob { StreamingJob::Table(_, table, ..) => table.owner, StreamingJob::Index(index, _) => index.owner, StreamingJob::Source(source) => source.owner, + StreamingJob::Subscription(subscription) => subscription.owner, } } @@ -242,16 +263,7 @@ impl StreamingJob { Self::Index(_, table) => table.definition.clone(), Self::Sink(sink, _) => sink.definition.clone(), Self::Source(source) => source.definition.clone(), - } - } - - pub fn properties(&self) -> HashMap { - match self { - Self::MaterializedView(table) => table.properties.clone(), - Self::Sink(sink, _) => sink.properties.clone(), - Self::Table(_, table, ..) => table.properties.clone(), - Self::Index(_, index_table) => index_table.properties.clone(), - Self::Source(source) => source.with_properties.clone(), + Self::Subscription(subscription) => subscription.definition.clone(), } } @@ -289,6 +301,7 @@ impl StreamingJob { vec![] } StreamingJob::Source(_) => vec![], + Self::Subscription(subscription) => subscription.dependent_relations.clone(), } } diff --git a/src/meta/src/manager/system_param/mod.rs b/src/meta/src/manager/system_param/mod.rs index 14d0e311a2d89..bdd54d3ed7b55 100644 --- a/src/meta/src/manager/system_param/mod.rs +++ b/src/meta/src/manager/system_param/mod.rs @@ -90,7 +90,12 @@ impl SystemParamsManager { let params = params_guard.deref_mut(); let mut mem_txn = VarTransaction::new(params); - set_system_param(mem_txn.deref_mut(), name, value).map_err(MetaError::system_params)?; + let Some((_new_value, diff)) = + set_system_param(mem_txn.deref_mut(), name, value).map_err(MetaError::system_params)? + else { + // No changes on the parameter. + return Ok(params.clone()); + }; let mut store_txn = Transaction::default(); mem_txn.apply_to_txn(&mut store_txn).await?; @@ -98,10 +103,10 @@ impl SystemParamsManager { mem_txn.commit(); - // TODO: check if the parameter is actually changed. - // Run common handler. - self.common_handler.handle_change(params.clone().into()); + self.common_handler.handle_change(&diff); + + // TODO: notify the diff instead of the snapshot. // Sync params to other managers on the meta node only once, since it's infallible. self.notification_manager @@ -111,7 +116,7 @@ impl SystemParamsManager { .await; // Sync params to worker nodes. - self.notify_workers(params).await; + self.notify_workers(params); Ok(params.clone()) } @@ -137,9 +142,7 @@ impl SystemParamsManager { return; } } - system_params_manager - .notify_workers(&*system_params_manager.params.read().await) - .await; + system_params_manager.notify_workers(&*system_params_manager.params.read().await); } }); @@ -147,16 +150,16 @@ impl SystemParamsManager { } // Notify workers of parameter change. - async fn notify_workers(&self, params: &SystemParams) { + // TODO: add system params into snapshot to avoid periodically sync. + fn notify_workers(&self, params: &SystemParams) { self.notification_manager - .notify_frontend(Operation::Update, Info::SystemParams(params.clone())) - .await; + .notify_frontend_without_version(Operation::Update, Info::SystemParams(params.clone())); self.notification_manager - .notify_compute(Operation::Update, Info::SystemParams(params.clone())) - .await; - self.notification_manager - .notify_compactor(Operation::Update, Info::SystemParams(params.clone())) - .await; + .notify_compute_without_version(Operation::Update, Info::SystemParams(params.clone())); + self.notification_manager.notify_compactor_without_version( + Operation::Update, + Info::SystemParams(params.clone()), + ); } } @@ -168,7 +171,7 @@ impl SystemParamsManager { // 4. None, None: A new version of RW cluster is launched for the first time and newly introduced // params are not set. The new field is not initialized either, just leave it as `None`. macro_rules! impl_merge_params { - ($({ $field:ident, $type:ty, $default:expr, $is_mutable:expr },)*) => { + ($({ $field:ident, $($rest:tt)* },)*) => { fn merge_params(mut persisted: SystemParams, init: SystemParams) -> SystemParams { $( match (persisted.$field.as_ref(), init.$field) { diff --git a/src/meta/src/model/catalog.rs b/src/meta/src/model/catalog.rs index 5d0d00a037ec2..76477aa66ab35 100644 --- a/src/meta/src/model/catalog.rs +++ b/src/meta/src/model/catalog.rs @@ -13,7 +13,7 @@ // limitations under the License. use risingwave_pb::catalog::{ - Connection, Database, Function, Index, Schema, Sink, Source, Table, View, + Connection, Database, Function, Index, Schema, Sink, Source, Subscription, Table, View, }; use crate::model::{MetadataModel, MetadataModelResult}; @@ -36,6 +36,8 @@ const CATALOG_TABLE_CF_NAME: &str = "cf/catalog_table"; const CATALOG_SCHEMA_CF_NAME: &str = "cf/catalog_schema"; /// Column family name for database catalog. const CATALOG_DATABASE_CF_NAME: &str = "cf/catalog_database"; +/// Column family name for database catalog. +const CATALOG_SUBSCRIPTION_CF_NAME: &str = "cf/catalog_subscription"; macro_rules! impl_model_for_catalog { ($name:ident, $cf:ident, $key_ty:ty, $key_fn:ident) => { @@ -71,6 +73,7 @@ impl_model_for_catalog!(Function, CATALOG_FUNCTION_CF_NAME, u32, get_id); impl_model_for_catalog!(Table, CATALOG_TABLE_CF_NAME, u32, get_id); impl_model_for_catalog!(Schema, CATALOG_SCHEMA_CF_NAME, u32, get_id); impl_model_for_catalog!(Database, CATALOG_DATABASE_CF_NAME, u32, get_id); +impl_model_for_catalog!(Subscription, CATALOG_SUBSCRIPTION_CF_NAME, u32, get_id); #[cfg(test)] mod tests { @@ -90,7 +93,7 @@ mod tests { #[tokio::test] async fn test_database() -> MetadataModelResult<()> { let env = MetaSrvEnv::for_test().await; - let store = env.meta_store(); + let store = env.meta_store_checked(); let databases = Database::list(store).await?; assert!(databases.is_empty()); assert!(Database::select(store, &0).await.unwrap().is_none()); diff --git a/src/meta/src/model/mod.rs b/src/meta/src/model/mod.rs index 53b78c671c660..8b53d87160efa 100644 --- a/src/meta/src/model/mod.rs +++ b/src/meta/src/model/mod.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod barrier; mod catalog; mod cluster; mod error; @@ -28,7 +27,6 @@ use std::marker::PhantomData; use std::ops::{Deref, DerefMut}; use async_trait::async_trait; -pub use barrier::*; pub use cluster::*; pub use error::*; pub use migration_plan::*; @@ -174,6 +172,7 @@ macro_rules! for_all_metadata_models { { risingwave_pb::catalog::Table }, { risingwave_pb::catalog::Index }, { risingwave_pb::catalog::Sink }, + { risingwave_pb::catalog::Subscription }, { risingwave_pb::catalog::Source }, { risingwave_pb::catalog::View }, { crate::model::stream::TableFragments }, diff --git a/src/meta/src/model/notification.rs b/src/meta/src/model/notification.rs index be3784836e5b0..0ea30bd61d7bf 100644 --- a/src/meta/src/model/notification.rs +++ b/src/meta/src/model/notification.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +use thiserror_ext::AsReport; + use crate::storage::{MetaStore, MetaStoreError, DEFAULT_COLUMN_FAMILY}; /// `NotificationVersion` records the last sent notification version, this will be stored @@ -31,7 +33,7 @@ impl NotificationVersion { { Ok(byte_vec) => memcomparable::from_slice(&byte_vec).unwrap(), Err(MetaStoreError::ItemNotFound(_)) => 0, - Err(e) => panic!("{:?}", e), + Err(e) => panic!("{}", e.as_report()), }; Self(version) } diff --git a/src/meta/src/model/stream.rs b/src/meta/src/model/stream.rs index 139581d46f43f..ef55f78493f85 100644 --- a/src/meta/src/model/stream.rs +++ b/src/meta/src/model/stream.rs @@ -23,14 +23,14 @@ use risingwave_pb::common::{ParallelUnit, ParallelUnitMapping}; use risingwave_pb::meta::table_fragments::actor_status::ActorState; use risingwave_pb::meta::table_fragments::{ActorStatus, Fragment, State}; use risingwave_pb::meta::table_parallelism::{ - FixedParallelism, Parallelism, PbAutoParallelism, PbCustomParallelism, PbFixedParallelism, + FixedParallelism, Parallelism, PbAdaptiveParallelism, PbCustomParallelism, PbFixedParallelism, PbParallelism, }; use risingwave_pb::meta::{PbTableFragments, PbTableParallelism}; use risingwave_pb::plan_common::PbExprContext; use risingwave_pb::stream_plan::stream_node::NodeBody; use risingwave_pb::stream_plan::{ - FragmentTypeFlag, PbStreamContext, StreamActor, StreamNode, StreamSource, + FragmentTypeFlag, PbFragmentTypeFlag, PbStreamContext, StreamActor, StreamNode, StreamSource, }; use super::{ActorId, FragmentId}; @@ -41,10 +41,20 @@ use crate::stream::{build_actor_connector_splits, build_actor_split_impls, Split /// Column family name for table fragments. const TABLE_FRAGMENTS_CF_NAME: &str = "cf/table_fragments"; +/// The parallelism for a `TableFragments`. #[derive(Debug, Copy, Clone, Eq, PartialEq)] pub enum TableParallelism { - Auto, + /// This is when the system decides the parallelism, based on the available parallel units. + Adaptive, + /// We set this when the `TableFragments` parallelism is changed. + /// All fragments which are part of the `TableFragment` will have the same parallelism as this. Fixed(usize), + /// We set this when the individual parallelisms of the `Fragments` + /// can differ within a `TableFragments`. + /// This is set for `risectl`, since it has a low-level interface, + /// scale individual `Fragments` within `TableFragments`. + /// When that happens, the `TableFragments` no longer has a consistent + /// parallelism, so we set this to indicate that. Custom, } @@ -53,7 +63,7 @@ impl From for TableParallelism { use Parallelism::*; match &value.parallelism { Some(Fixed(FixedParallelism { parallelism: n })) => Self::Fixed(*n as usize), - Some(Auto(_)) => Self::Auto, + Some(Adaptive(_)) | Some(Auto(_)) => Self::Adaptive, Some(Custom(_)) => Self::Custom, _ => unreachable!(), } @@ -65,7 +75,7 @@ impl From for PbTableParallelism { use TableParallelism::*; let parallelism = match value { - Auto => PbParallelism::Auto(PbAutoParallelism {}), + Adaptive => PbParallelism::Adaptive(PbAdaptiveParallelism {}), Fixed(n) => PbParallelism::Fixed(PbFixedParallelism { parallelism: n as u32, }), @@ -188,7 +198,7 @@ impl TableFragments { fragments, &BTreeMap::new(), StreamContext::default(), - TableParallelism::Auto, + TableParallelism::Adaptive, ) } @@ -337,14 +347,17 @@ impl TableFragments { /// Returns barrier inject actor ids. pub fn barrier_inject_actor_ids(&self) -> Vec { - Self::filter_actor_ids(self, |fragment_type_mask| { - (fragment_type_mask - & (FragmentTypeFlag::Source as u32 - | FragmentTypeFlag::Now as u32 - | FragmentTypeFlag::Values as u32 - | FragmentTypeFlag::BarrierRecv as u32)) - != 0 - }) + Self::filter_actor_ids(self, Self::is_injectable) + } + + /// Check if the fragment type mask is injectable. + pub fn is_injectable(fragment_type_mask: u32) -> bool { + (fragment_type_mask + & (PbFragmentTypeFlag::Source as u32 + | PbFragmentTypeFlag::Now as u32 + | PbFragmentTypeFlag::Values as u32 + | PbFragmentTypeFlag::BarrierRecv as u32)) + != 0 } /// Returns mview actor ids. diff --git a/src/meta/src/rpc/ddl_controller.rs b/src/meta/src/rpc/ddl_controller.rs index 1ab3aad2ddf3d..7d16d353643fe 100644 --- a/src/meta/src/rpc/ddl_controller.rs +++ b/src/meta/src/rpc/ddl_controller.rs @@ -23,6 +23,7 @@ use itertools::Itertools; use rand::Rng; use risingwave_common::config::DefaultParallelism; use risingwave_common::hash::{ParallelUnitMapping, VirtualNode}; +use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_common::util::epoch::Epoch; use risingwave_common::util::stream_graph_visitor::{ @@ -30,6 +31,7 @@ use risingwave_common::util::stream_graph_visitor::{ }; use risingwave_common::{bail, current_cluster_version}; use risingwave_connector::dispatch_source_prop; +use risingwave_connector::error::ConnectorError; use risingwave_connector::source::cdc::CdcSourceType; use risingwave_connector::source::{ ConnectorProperties, SourceEnumeratorContext, SourceProperties, SplitEnumerator, @@ -56,25 +58,25 @@ use risingwave_pb::stream_plan::{ Dispatcher, DispatcherType, FragmentTypeFlag, MergeNode, PbStreamFragmentGraph, StreamFragmentGraph as StreamFragmentGraphProto, }; +use thiserror_ext::AsReport; use tokio::sync::Semaphore; use tokio::time::sleep; use tracing::log::warn; use tracing::Instrument; use crate::barrier::BarrierManagerRef; -use crate::controller::catalog::{CatalogControllerRef, ReleaseContext}; use crate::manager::{ CatalogManagerRef, ConnectionId, DatabaseId, FragmentManagerRef, FunctionId, IdCategory, IdCategoryType, IndexId, LocalNotification, MetaSrvEnv, MetadataManager, MetadataManagerV1, NotificationVersion, RelationIdEnum, SchemaId, SinkId, SourceId, StreamingClusterInfo, - StreamingJob, TableId, UserId, ViewId, IGNORED_NOTIFICATION_VERSION, + StreamingJob, SubscriptionId, TableId, UserId, ViewId, IGNORED_NOTIFICATION_VERSION, }; use crate::model::{FragmentId, StreamContext, TableFragments, TableParallelism}; use crate::rpc::cloud_provider::AwsEc2Client; use crate::stream::{ validate_sink, ActorGraphBuildResult, ActorGraphBuilder, CompleteStreamFragmentGraph, - CreateStreamingJobContext, GlobalStreamManagerRef, ReplaceTableContext, SourceManagerRef, - StreamFragmentGraph, + CreateStreamingJobContext, CreateStreamingJobOption, GlobalStreamManagerRef, + ReplaceTableContext, SourceManagerRef, StreamFragmentGraph, }; use crate::{MetaError, MetaResult}; @@ -99,6 +101,7 @@ pub enum StreamingJobId { Sink(SinkId), Table(Option, TableId), Index(IndexId), + Subscription(SubscriptionId), } impl StreamingJobId { @@ -107,6 +110,7 @@ impl StreamingJobId { match self { StreamingJobId::MaterializedView(id) | StreamingJobId::Sink(id) + | StreamingJobId::Subscription(id) | StreamingJobId::Table(_, id) | StreamingJobId::Index(id) => *id, } @@ -268,7 +272,7 @@ impl DdlController { /// would be a huge hassle and pain if we don't spawn here. pub async fn run_command(&self, command: DdlCommand) -> MetaResult { if !command.allow_in_recovery() { - self.barrier_manager.check_status_running().await?; + self.barrier_manager.check_status_running()?; } let ctrl = self.clone(); let fut = async move { @@ -352,9 +356,10 @@ impl DdlController { &self, table_id: u32, parallelism: PbTableParallelism, + deferred: bool, ) -> MetaResult<()> { self.stream_manager - .alter_table_parallelism(table_id, parallelism.into()) + .alter_table_parallelism(table_id, parallelism.into(), deferred) .await } @@ -380,46 +385,20 @@ impl DdlController { Ok(version) } - async fn drop_database_v2( - &self, - catalog_controller: &CatalogControllerRef, - database_id: DatabaseId, - ) -> MetaResult { - let ( - ReleaseContext { - streaming_jobs, - source_ids, - connections, - .. - }, - version, - ) = catalog_controller.drop_database(database_id as _).await?; - self.source_manager - .unregister_sources(source_ids.into_iter().map(|id| id as _).collect()) - .await; - self.stream_manager - .drop_streaming_jobs( - streaming_jobs - .into_iter() - .map(|id| (id as u32).into()) - .collect(), - ) - .await; - for svc in connections { - self.delete_vpc_endpoint_v2(svc.into_inner()).await?; - } - Ok(version) - } - async fn drop_database(&self, database_id: DatabaseId) -> MetaResult { match &self.metadata_manager { MetadataManager::V1(mgr) => { self.drop_database_v1(&mgr.catalog_manager, database_id) .await } - MetadataManager::V2(mgr) => { - self.drop_database_v2(&mgr.catalog_controller, database_id) - .await + MetadataManager::V2(_) => { + self.drop_object( + ObjectType::Database, + database_id as _, + DropMode::Cascade, + None, + ) + .await } } } @@ -437,9 +416,8 @@ impl DdlController { async fn drop_schema(&self, schema_id: SchemaId) -> MetaResult { match &self.metadata_manager { MetadataManager::V1(mgr) => mgr.catalog_manager.drop_schema(schema_id).await, - MetadataManager::V2(mgr) => { - mgr.catalog_controller - .drop_schema(schema_id as _, DropMode::Restrict) + MetadataManager::V2(_) => { + self.drop_object(ObjectType::Schema, schema_id as _, DropMode::Restrict, None) .await } } @@ -461,7 +439,7 @@ impl DdlController { mgr.catalog_manager .cancel_create_source_procedure(&source) .await?; - return Err(e.into()); + return Err(e); } mgr.catalog_manager @@ -482,7 +460,9 @@ impl DdlController { drop_mode: DropMode, ) -> MetaResult { let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support drop source in v2"); + return self + .drop_object(ObjectType::Source, source_id as _, drop_mode, None) + .await; }; // 1. Drop source in catalog. let (version, streaming_job_ids) = mgr @@ -550,7 +530,9 @@ impl DdlController { drop_mode: DropMode, ) -> MetaResult { let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support drop view in v2"); + return self + .drop_object(ObjectType::View, view_id as _, drop_mode, None) + .await; }; let (version, streaming_job_ids) = mgr .catalog_manager @@ -583,20 +565,26 @@ impl DdlController { &self, connection_id: ConnectionId, ) -> MetaResult { - let (version, connection) = match &self.metadata_manager { - MetadataManager::V1(mgr) => mgr.catalog_manager.drop_connection(connection_id).await?, - MetadataManager::V2(mgr) => { - mgr.catalog_controller - .drop_connection(connection_id as _) - .await? + match &self.metadata_manager { + MetadataManager::V1(mgr) => { + let (version, connection) = + mgr.catalog_manager.drop_connection(connection_id).await?; + self.delete_vpc_endpoint(&connection).await?; + Ok(version) } - }; - - self.delete_vpc_endpoint(&connection).await?; - Ok(version) + MetadataManager::V2(_) => { + self.drop_object( + ObjectType::Connection, + connection_id as _, + DropMode::Restrict, + None, + ) + .await + } + } } - async fn delete_vpc_endpoint(&self, connection: &Connection) -> MetaResult<()> { + pub(crate) async fn delete_vpc_endpoint(&self, connection: &Connection) -> MetaResult<()> { // delete AWS vpc endpoint if let Some(connection::Info::PrivateLinkService(svc)) = &connection.info && svc.get_provider()? == PbPrivateLinkProvider::Aws @@ -613,7 +601,7 @@ impl DdlController { Ok(()) } - async fn delete_vpc_endpoint_v2(&self, svc: PrivateLinkService) -> MetaResult<()> { + pub(crate) async fn delete_vpc_endpoint_v2(&self, svc: PrivateLinkService) -> MetaResult<()> { // delete AWS vpc endpoint if svc.get_provider()? == PbPrivateLinkProvider::Aws { if let Some(aws_cli) = self.aws_client.as_ref() { @@ -637,7 +625,7 @@ impl DdlController { ) -> MetaResult { let MetadataManager::V1(mgr) = &self.metadata_manager else { return self - .create_streaming_job_v2(stream_job, fragment_graph) + .create_streaming_job_v2(stream_job, fragment_graph, affected_table_replace_info) .await; }; let id = self.gen_unique_id::<{ IdCategory::Table }>().await?; @@ -675,7 +663,7 @@ impl DdlController { .acquire() .await .unwrap(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let stream_ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap()); @@ -695,9 +683,32 @@ impl DdlController { mgr.catalog_manager .start_create_stream_job_procedure(&stream_job, internal_tables.clone()) .await?; + let affected_table_replace_info = match affected_table_replace_info { + Some(replace_table_info) => { + let MetadataManager::V1(mgr) = &self.metadata_manager else { + unimplemented!("support replace table in v2"); + }; + + let ReplaceTableInfo { + mut streaming_job, + fragment_graph, + .. + } = replace_table_info; + let fragment_graph = self + .prepare_replace_table( + mgr.catalog_manager.clone(), + &mut streaming_job, + fragment_graph, + ) + .await?; + + Some((streaming_job, fragment_graph)) + } + None => None, + }; // 4. Build and persist stream job. - let result = try { + let result: MetaResult<_> = try { tracing::debug!(id = stream_job.id(), "building stream job"); let (ctx, table_fragments) = self .build_stream_job( @@ -745,6 +756,7 @@ impl DdlController { let (ctx, table_fragments) = match result { Ok(r) => r, Err(e) => { + tracing::error!(error = %e.as_report(), id = stream_job.id(), "failed to create streaming job"); self.cancel_stream_job(&stream_job, internal_tables, Some(&e)) .await?; return Err(e); @@ -778,7 +790,7 @@ impl DdlController { .await; match result { Err(e) => { - tracing::error!(id=stream_job_id, error = ?e, "finish stream job failed") + tracing::error!(id = stream_job_id, error = %e.as_report(), "finish stream job failed") } Ok(_) => { tracing::info!(id = stream_job_id, "finish stream job succeeded") @@ -795,7 +807,7 @@ impl DdlController { pub(crate) async fn validate_cdc_table( table: &Table, table_fragments: &TableFragments, - ) -> anyhow::Result<()> { + ) -> MetaResult<()> { let stream_scan_fragment = table_fragments .fragments .values() @@ -811,7 +823,7 @@ impl DdlController { async fn new_enumerator_for_validate( source_props: P, - ) -> Result { + ) -> Result { P::SplitEnumerator::new(source_props, SourceEnumeratorContext::default().into()).await } @@ -840,29 +852,19 @@ impl DdlController { // Here we modify the union node of the downstream table by the TableFragments of the to-be-created sink upstream. // The merge in the union has already been set up in the frontend and will be filled with specific upstream actors in this function. // Meanwhile, the Dispatcher corresponding to the upstream of the merge will also be added to the replace table context here. - async fn inject_replace_table_job_for_table_sink( + pub(crate) async fn inject_replace_table_job_for_table_sink( &self, - mgr: &MetadataManagerV1, + dummy_id: u32, + mgr: &MetadataManager, stream_ctx: StreamContext, sink: Option<&Sink>, creating_sink_table_fragments: Option<&TableFragments>, dropping_sink_id: Option, - ReplaceTableInfo { - mut streaming_job, - fragment_graph, - .. - }: ReplaceTableInfo, - ) -> MetaResult<(StreamingJob, ReplaceTableContext, TableFragments)> { - let fragment_graph = self - .prepare_replace_table( - mgr.catalog_manager.clone(), - &mut streaming_job, - fragment_graph, - ) - .await?; - + streaming_job: &StreamingJob, + fragment_graph: StreamFragmentGraph, + ) -> MetaResult<(ReplaceTableContext, TableFragments)> { let (mut replace_table_ctx, mut table_fragments) = self - .build_replace_table(mgr, stream_ctx, &streaming_job, fragment_graph, None) + .build_replace_table(stream_ctx, streaming_job, fragment_graph, None, dummy_id) .await?; let mut union_fragment_id = None; @@ -903,17 +905,14 @@ impl DdlController { } let [table_catalog]: [_; 1] = mgr - .catalog_manager - .get_tables(&[table.id]) - .await + .get_table_catalog_by_ids(vec![table.id]) + .await? .try_into() .expect("Target table should exist in sink into table"); assert_eq!(table_catalog.incoming_sinks, table.incoming_sinks); { - let guard = mgr.fragment_manager.get_fragment_read_guard().await; - for sink_id in &table_catalog.incoming_sinks { if let Some(dropping_sink_id) = dropping_sink_id && *sink_id == dropping_sink_id @@ -921,10 +920,9 @@ impl DdlController { continue; }; - let sink_table_fragments = guard - .table_fragments() - .get(&risingwave_common::catalog::TableId::new(*sink_id)) - .unwrap(); + let sink_table_fragments = mgr + .get_job_fragments_by_id(&risingwave_common::catalog::TableId::new(*sink_id)) + .await?; let sink_fragment = sink_table_fragments.sink_fragment().unwrap(); @@ -952,7 +950,7 @@ impl DdlController { } } - Ok((streaming_job, replace_table_ctx, table_fragments)) + Ok((replace_table_ctx, table_fragments)) } fn inject_replace_table_plan_for_sink( @@ -1073,7 +1071,7 @@ impl DdlController { let job_id = stream_job.id(); tracing::debug!(id = job_id, "creating stream job"); - let result = try { + let result: MetaResult<()> = try { // Add table fragments to meta store with state: `State::Initial`. mgr.fragment_manager .start_create_table_fragments(table_fragments.clone()) @@ -1087,7 +1085,7 @@ impl DdlController { if let Err(e) = result { match stream_job.create_type() { CreateType::Background => { - tracing::error!(id = job_id, error = ?e, "finish stream job failed"); + tracing::error!(id = job_id, error = %e.as_report(), "finish stream job failed"); let should_cancel = match mgr .fragment_manager .select_table_fragments_by_table_id(&job_id.into()) @@ -1129,10 +1127,37 @@ impl DdlController { drop_mode: DropMode, target_replace_info: Option, ) -> MetaResult { - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support drop streaming job in v2"); - }; - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + match &self.metadata_manager { + MetadataManager::V1(_) => { + self.drop_streaming_job_v1(job_id, drop_mode, target_replace_info) + .await + } + MetadataManager::V2(_) => { + let (object_id, object_type) = match job_id { + StreamingJobId::MaterializedView(id) => (id as _, ObjectType::Table), + StreamingJobId::Sink(id) => (id as _, ObjectType::Sink), + StreamingJobId::Table(_, id) => (id as _, ObjectType::Table), + StreamingJobId::Index(idx) => (idx as _, ObjectType::Index), + StreamingJobId::Subscription(id) => (id as _, ObjectType::Sink), + }; + + let version = self + .drop_object(object_type, object_id, drop_mode, target_replace_info) + .await?; + + Ok(version) + } + } + } + + async fn drop_streaming_job_v1( + &self, + job_id: StreamingJobId, + drop_mode: DropMode, + target_replace_info: Option, + ) -> MetaResult { + let mgr = self.metadata_manager.as_v1_ref(); + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let (mut version, streaming_job_ids) = match job_id { StreamingJobId::MaterializedView(table_id) => { mgr.catalog_manager @@ -1171,6 +1196,15 @@ impl DdlController { ) .await? } + StreamingJobId::Subscription(subscription_id) => { + mgr.catalog_manager + .drop_relation( + RelationIdEnum::Subscription(subscription_id), + mgr.fragment_manager.clone(), + drop_mode, + ) + .await? + } }; if let Some(replace_table_info) = target_replace_info { @@ -1181,35 +1215,69 @@ impl DdlController { panic!("additional replace table event only occurs when dropping sink into table") }; - let (streaming_job, context, table_fragments) = self - .inject_replace_table_job_for_table_sink( - mgr, - stream_ctx, - None, - None, - Some(sink_id), - replace_table_info, + let ReplaceTableInfo { + mut streaming_job, + fragment_graph, + .. + } = replace_table_info; + let fragment_graph = self + .prepare_replace_table( + mgr.catalog_manager.clone(), + &mut streaming_job, + fragment_graph, ) .await?; - // Add table fragments to meta store with state: `State::Initial`. - mgr.fragment_manager - .start_create_table_fragments(table_fragments.clone()) - .await?; + let result: MetaResult<()> = try { + tracing::debug!(id = streaming_job.id(), "replacing table for dropped sink"); - self.stream_manager - .replace_table(table_fragments, context) - .await?; + let dummy_id = self + .env + .id_gen_manager() + .generate::<{ IdCategory::Table }>() + .await? as u32; - version = self - .finish_replace_table( - mgr.catalog_manager.clone(), - &streaming_job, - None, - None, - Some(sink_id), - ) - .await?; + let (context, table_fragments) = self + .inject_replace_table_job_for_table_sink( + dummy_id, + &self.metadata_manager, + stream_ctx, + None, + None, + Some(sink_id), + &streaming_job, + fragment_graph, + ) + .await?; + + // Add table fragments to meta store with state: `State::Initial`. + mgr.fragment_manager + .start_create_table_fragments(table_fragments.clone()) + .await?; + + self.stream_manager + .replace_table(table_fragments, context) + .await?; + }; + + match result { + Ok(_) => { + version = self + .finish_replace_table( + mgr.catalog_manager.clone(), + &streaming_job, + None, + None, + Some(sink_id), + ) + .await?; + } + Err(err) => { + tracing::error!(error = %err.as_report(), "failed to replace table for dropped sink"); + self.cancel_replace_table(mgr.catalog_manager.clone(), &streaming_job) + .await?; + } + } } self.stream_manager @@ -1267,7 +1335,7 @@ impl DdlController { stream_ctx: StreamContext, stream_job: &StreamingJob, fragment_graph: StreamFragmentGraph, - affected_table_replace_info: Option, + affected_table_replace_info: Option<(StreamingJob, StreamFragmentGraph)>, ) -> MetaResult<(CreateStreamingJobContext, TableFragments)> { let id = stream_job.id(); let default_parallelism = fragment_graph.default_parallelism(); @@ -1322,7 +1390,7 @@ impl DdlController { // actors on the compute nodes. let table_parallelism = match default_parallelism { - None => TableParallelism::Auto, + None => TableParallelism::Adaptive, Some(parallelism) => TableParallelism::Fixed(parallelism.get()), }; @@ -1335,25 +1403,45 @@ impl DdlController { ); let replace_table_job_info = match affected_table_replace_info { - Some(replace_table_info) => { + Some((streaming_job, fragment_graph)) => { let StreamingJob::Sink(s, _) = stream_job else { bail!("additional replace table event only occurs when sinking into table"); }; - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support replace table in v2"); + + let dummy_id = match &self.metadata_manager { + MetadataManager::V1(_) => { + self.env + .id_gen_manager() + .generate::<{ IdCategory::Table }>() + .await? as u32 + } + MetadataManager::V2(mgr) => { + let table = streaming_job.table().unwrap(); + mgr.catalog_controller + .create_job_catalog_for_replace( + &streaming_job, + &stream_ctx, + table.get_version()?, + &fragment_graph.default_parallelism(), + ) + .await? as u32 + } }; - Some( - self.inject_replace_table_job_for_table_sink( - mgr, + let (context, table_fragments) = self + .inject_replace_table_job_for_table_sink( + dummy_id, + &self.metadata_manager, stream_ctx, Some(s), Some(&table_fragments), None, - replace_table_info, + &streaming_job, + fragment_graph, ) - .await?, - ) + .await?; + + Some((streaming_job, context, table_fragments)) } None => None, }; @@ -1364,12 +1452,15 @@ impl DdlController { internal_tables, building_locations, existing_locations, - table_properties: stream_job.properties(), definition: stream_job.definition(), mv_table_id: stream_job.mv_table(), create_type: stream_job.create_type(), ddl_type: stream_job.into(), replace_table_job_info, + // TODO: https://github.com/risingwavelabs/risingwave/issues/14793 + option: CreateStreamingJobOption { + new_independent_compaction_group: false, + }, }; // 4. Mark tables as creating, including internal tables and the table of the stream job. @@ -1396,9 +1487,7 @@ impl DdlController { internal_tables: Vec
, error: Option<&impl ToString>, ) -> MetaResult<()> { - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support cancel streaming job in v2"); - }; + let mgr = self.metadata_manager.as_v1_ref(); let error = error.map(ToString::to_string).unwrap_or_default(); let event = risingwave_pb::meta::event_log::EventCreateStreamJobFail { id: stream_job.id(), @@ -1422,7 +1511,10 @@ impl DdlController { .await; creating_internal_table_ids.push(table.id); if let Err(e) = result { - tracing::warn!("Failed to cancel create table procedure, perhaps barrier manager has already cleaned it. Reason: {e:#?}"); + tracing::warn!( + error = %e.as_report(), + "Failed to cancel create table procedure, perhaps barrier manager has already cleaned it." + ); } } StreamingJob::Sink(sink, target_table) => { @@ -1430,6 +1522,11 @@ impl DdlController { .cancel_create_sink_procedure(sink, target_table) .await; } + StreamingJob::Subscription(subscription) => { + mgr.catalog_manager + .cancel_create_subscription_procedure(subscription) + .await; + } StreamingJob::Table(source, table, ..) => { if let Some(source) = source { mgr.catalog_manager @@ -1444,7 +1541,10 @@ impl DdlController { ) .await; if let Err(e) = result { - tracing::warn!("Failed to cancel create table procedure, perhaps barrier manager has already cleaned it. Reason: {e:#?}"); + tracing::warn!( + error = %e.as_report(), + "Failed to cancel create table procedure, perhaps barrier manager has already cleaned it." + ); } } creating_internal_table_ids.push(table.id); @@ -1513,6 +1613,11 @@ impl DdlController { version } + StreamingJob::Subscription(subscription) => { + mgr.catalog_manager + .finish_create_subscription_procedure(internal_tables, subscription) + .await? + } StreamingJob::Table(source, table, ..) => { creating_internal_table_ids.push(table.id); if let Some(source) = source { @@ -1586,23 +1691,30 @@ impl DdlController { table_col_index_mapping: Option, ) -> MetaResult { let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support replace table in v2"); + return self + .replace_table_v2(stream_job, fragment_graph, table_col_index_mapping) + .await; }; - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; let stream_ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap()); let fragment_graph = self .prepare_replace_table(mgr.catalog_manager.clone(), &mut stream_job, fragment_graph) .await?; + let dummy_id = self + .env + .id_gen_manager() + .generate::<{ IdCategory::Table }>() + .await? as u32; - let result = try { + let result: MetaResult<()> = try { let (ctx, table_fragments) = self .build_replace_table( - mgr, stream_ctx, &stream_job, fragment_graph, table_col_index_mapping.clone(), + dummy_id, ) .await?; @@ -1628,6 +1740,7 @@ impl DdlController { .await } Err(err) => { + tracing::error!(error = %err.as_report(), "failed to replace table"); self.cancel_replace_table(mgr.catalog_manager.clone(), &stream_job) .await?; Err(err) @@ -1663,39 +1776,41 @@ impl DdlController { /// `build_replace_table` builds a table replacement and returns the context and new table /// fragments. - async fn build_replace_table( + /// + /// Note that we use a dummy ID for the new table fragments and replace it with the real one after + /// replacement is finished. + pub(crate) async fn build_replace_table( &self, - mgr: &MetadataManagerV1, stream_ctx: StreamContext, stream_job: &StreamingJob, mut fragment_graph: StreamFragmentGraph, table_col_index_mapping: Option, + dummy_table_id: TableId, ) -> MetaResult<(ReplaceTableContext, TableFragments)> { let id = stream_job.id(); let default_parallelism = fragment_graph.default_parallelism(); let expr_context = stream_ctx.to_expr_context(); - let old_table_fragments = mgr - .fragment_manager - .select_table_fragments_by_table_id(&id.into()) + let old_table_fragments = self + .metadata_manager + .get_job_fragments_by_id(&id.into()) .await?; let old_internal_table_ids = old_table_fragments.internal_table_ids(); - let old_internal_tables = mgr - .catalog_manager - .get_tables(&old_internal_table_ids) - .await; + let old_internal_tables = self + .metadata_manager + .get_table_catalog_by_ids(old_internal_table_ids) + .await?; fragment_graph.fit_internal_table_ids(old_internal_tables)?; // 1. Resolve the edges to the downstream fragments, extend the fragment graph to a complete // graph that contains all information needed for building the actor graph. - let original_table_fragment = mgr.fragment_manager.get_mview_fragment(id.into()).await?; + let original_table_fragment = old_table_fragments + .mview_fragment() + .expect("mview fragment not found"); // Map the column indices in the dispatchers with the given mapping. - let downstream_fragments = mgr - .fragment_manager - .get_downstream_fragments(id.into()) - .await? + let downstream_fragments = self.metadata_manager.get_downstream_chain_fragments(id).await? .into_iter() .map(|(d, f)| if let Some(mapping) = &table_col_index_mapping { @@ -1719,7 +1834,7 @@ impl DdlController { )?; // 2. Build the actor graph. - let cluster_info = mgr.cluster_manager.get_streaming_cluster_info().await; + let cluster_info = self.metadata_manager.get_streaming_cluster_info().await?; let parallelism = self.resolve_stream_parallelism(default_parallelism, &cluster_info)?; let actor_graph_builder = @@ -1736,27 +1851,16 @@ impl DdlController { .await?; assert!(dispatchers.is_empty()); - // 3. Assign a new dummy ID for the new table fragments. - // - // FIXME: we use a dummy table ID for new table fragments, so we can drop the old fragments - // with the real table ID, then replace the dummy table ID with the real table ID. This is a - // workaround for not having the version info in the fragment manager. - let dummy_id = self - .env - .id_gen_manager() - .generate::<{ IdCategory::Table }>() - .await? as u32; - let table_parallelism = match default_parallelism { - None => TableParallelism::Auto, + None => TableParallelism::Adaptive, Some(parallelism) => TableParallelism::Fixed(parallelism.get()), }; - // 4. Build the table fragments structure that will be persisted in the stream manager, and + // 3. Build the table fragments structure that will be persisted in the stream manager, and // the context that contains all information needed for building the actors on the compute // nodes. let table_fragments = TableFragments::new( - dummy_id.into(), + dummy_table_id.into(), graph, &building_locations.actor_locations, stream_ctx, @@ -1770,7 +1874,6 @@ impl DdlController { dispatchers, building_locations, existing_locations, - table_properties: stream_job.properties(), }; Ok((ctx, table_fragments)) @@ -1847,6 +1950,11 @@ impl DdlController { .alter_database_name(database_id, new_name) .await } + alter_name_request::Object::SubscriptionId(sink_id) => { + mgr.catalog_manager + .alter_subscription_name(sink_id, new_name) + .await + } }, MetadataManager::V2(mgr) => { let (obj_type, id) = match relation { @@ -1863,6 +1971,9 @@ impl DdlController { alter_name_request::Object::DatabaseId(id) => { (ObjectType::Database, id as ObjectId) } + alter_name_request::Object::SubscriptionId(id) => { + (ObjectType::Subscription, id as ObjectId) + } }; mgr.catalog_controller .alter_name(obj_type, id, new_name) @@ -1890,6 +2001,7 @@ impl DdlController { Object::SinkId(id) => (ObjectType::Sink, id as ObjectId), Object::SchemaId(id) => (ObjectType::Schema, id as ObjectId), Object::DatabaseId(id) => (ObjectType::Database, id as ObjectId), + Object::SubscriptionId(id) => (ObjectType::Subscription, id as ObjectId), }; mgr.catalog_controller .alter_owner(obj_type, id, owner_id as _) @@ -1929,6 +2041,9 @@ impl DdlController { alter_set_schema_request::Object::ConnectionId(id) => { (ObjectType::Connection, id as ObjectId) } + alter_set_schema_request::Object::SubscriptionId(id) => { + (ObjectType::Subscription, id as ObjectId) + } }; mgr.catalog_controller .alter_schema(obj_type, id, new_schema_id as _) diff --git a/src/meta/src/rpc/ddl_controller_v2.rs b/src/meta/src/rpc/ddl_controller_v2.rs index 4a83914f4f08a..0ad4b78cdefd7 100644 --- a/src/meta/src/rpc/ddl_controller_v2.rs +++ b/src/meta/src/rpc/ddl_controller_v2.rs @@ -13,19 +13,25 @@ // limitations under the License. use itertools::Itertools; +use risingwave_common::util::column_index_mapping::ColIndexMapping; use risingwave_common::util::stream_graph_visitor::visit_fragment; +use risingwave_meta_model_v2::object::ObjectType; +use risingwave_meta_model_v2::ObjectId; use risingwave_pb::catalog::CreateType; use risingwave_pb::ddl_service::TableJobType; use risingwave_pb::stream_plan::stream_node::NodeBody; +use risingwave_pb::stream_plan::update_mutation::PbMergeUpdate; use risingwave_pb::stream_plan::StreamFragmentGraph as StreamFragmentGraphProto; use thiserror_ext::AsReport; +use crate::controller::catalog::ReleaseContext; use crate::manager::{ - MetadataManager, MetadataManagerV2, NotificationVersion, StreamingJob, - IGNORED_NOTIFICATION_VERSION, + MetadataManagerV2, NotificationVersion, StreamingJob, IGNORED_NOTIFICATION_VERSION, }; use crate::model::{MetadataModel, StreamContext}; -use crate::rpc::ddl_controller::{fill_table_stream_graph_info, DdlController}; +use crate::rpc::ddl_controller::{ + fill_table_stream_graph_info, DdlController, DropMode, ReplaceTableInfo, +}; use crate::stream::{validate_sink, StreamFragmentGraph}; use crate::MetaResult; @@ -34,14 +40,13 @@ impl DdlController { &self, mut streaming_job: StreamingJob, mut fragment_graph: StreamFragmentGraphProto, + affected_table_replace_info: Option, ) -> MetaResult { - let MetadataManager::V2(mgr) = &self.metadata_manager else { - unreachable!("MetadataManager should be V2") - }; + let mgr = self.metadata_manager.as_v2_ref(); let ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap()); mgr.catalog_controller - .create_job_catalog(&mut streaming_job, &ctx) + .create_job_catalog(&mut streaming_job, &ctx, &fragment_graph.parallelism) .await?; let job_id = streaming_job.id(); @@ -74,19 +79,34 @@ impl DdlController { .acquire() .await .unwrap(); - let _reschedule_job_lock = self.stream_manager.reschedule_lock.read().await; + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; // create streaming job. match self - .create_streaming_job_inner_v2(mgr, ctx, &mut streaming_job, fragment_graph) + .create_streaming_job_inner_v2( + mgr, + ctx, + &mut streaming_job, + fragment_graph, + affected_table_replace_info, + ) .await { Ok(version) => Ok(version), Err(err) => { tracing::error!(id = job_id, error = ?err.as_report(), "failed to create streaming job"); + let event = risingwave_pb::meta::event_log::EventCreateStreamJobFail { + id: streaming_job.id(), + name: streaming_job.name(), + definition: streaming_job.definition(), + error: err.as_report().to_string(), + }; + self.env.event_log_manager_ref().add_event_logs(vec![ + risingwave_pb::meta::event_log::Event::CreateStreamJobFail(event), + ]); let aborted = mgr .catalog_controller - .try_abort_creating_streaming_job(job_id as _) + .try_abort_creating_streaming_job(job_id as _, false) .await?; if aborted { tracing::warn!(id = job_id, "aborted streaming job"); @@ -108,6 +128,7 @@ impl DdlController { ctx: StreamContext, streaming_job: &mut StreamingJob, fragment_graph: StreamFragmentGraphProto, + affected_table_replace_info: Option, ) -> MetaResult { let mut fragment_graph = StreamFragmentGraph::new(&self.env, fragment_graph, streaming_job).await?; @@ -122,10 +143,34 @@ impl DdlController { .await?; fragment_graph.refill_internal_table_ids(table_id_map); + let affected_table_replace_info = match affected_table_replace_info { + Some(replace_table_info) => { + let ReplaceTableInfo { + mut streaming_job, + fragment_graph, + .. + } = replace_table_info; + + let fragment_graph = + StreamFragmentGraph::new(&self.env, fragment_graph, &streaming_job).await?; + streaming_job.set_table_fragment_id(fragment_graph.table_fragment_id()); + streaming_job.set_dml_fragment_id(fragment_graph.dml_fragment_id()); + let streaming_job = streaming_job; + + Some((streaming_job, fragment_graph)) + } + None => None, + }; + // create fragment and actor catalogs. tracing::debug!(id = streaming_job.id(), "building streaming job"); let (ctx, table_fragments) = self - .build_stream_job(ctx, streaming_job, fragment_graph, None) + .build_stream_job( + ctx, + streaming_job, + fragment_graph, + affected_table_replace_info, + ) .await?; match streaming_job { @@ -137,9 +182,12 @@ impl DdlController { self.source_manager.register_source(source).await?; } StreamingJob::Sink(sink, target_table) => { - if target_table.is_some() { - unimplemented!("support create sink into table in v2"); + if let Some((StreamingJob::Table(source, table, _), ..)) = + &ctx.replace_table_job_info + { + *target_table = Some((table.clone(), source.clone())); } + // Validate the sink on the connector node. validate_sink(sink).await?; } @@ -151,20 +199,46 @@ impl DdlController { } mgr.catalog_controller - .prepare_streaming_job(table_fragments.to_protobuf(), streaming_job) + .prepare_streaming_job(table_fragments.to_protobuf(), streaming_job, false) .await?; // create streaming jobs. let stream_job_id = streaming_job.id(); match streaming_job.create_type() { CreateType::Unspecified | CreateType::Foreground => { + let replace_table_job_info = ctx.replace_table_job_info.as_ref().map( + |(streaming_job, ctx, table_fragments)| { + ( + streaming_job.clone(), + ctx.merge_updates.clone(), + table_fragments.table_id(), + ) + }, + ); + self.stream_manager .create_streaming_job(table_fragments, ctx) .await?; - let version = mgr + + let mut version = mgr .catalog_controller .finish_streaming_job(stream_job_id as _) .await?; + + if let Some((streaming_job, merge_updates, table_id)) = replace_table_job_info { + version = mgr + .catalog_controller + .finish_replace_streaming_job( + table_id.table_id as _, + streaming_job, + merge_updates, + None, + Some(stream_job_id), + None, + ) + .await?; + } + Ok(version) } CreateType::Background => { @@ -191,4 +265,260 @@ impl DdlController { } } } + + pub async fn drop_object( + &self, + object_type: ObjectType, + object_id: ObjectId, + drop_mode: DropMode, + target_replace_info: Option, + ) -> MetaResult { + let mgr = self.metadata_manager.as_v2_ref(); + let (release_ctx, mut version) = match object_type { + ObjectType::Database => mgr.catalog_controller.drop_database(object_id).await?, + ObjectType::Schema => { + return mgr + .catalog_controller + .drop_schema(object_id, drop_mode) + .await; + } + ObjectType::Function => { + return mgr.catalog_controller.drop_function(object_id).await; + } + ObjectType::Connection => { + let (version, conn) = mgr.catalog_controller.drop_connection(object_id).await?; + self.delete_vpc_endpoint(&conn).await?; + return Ok(version); + } + _ => { + mgr.catalog_controller + .drop_relation(object_type, object_id, drop_mode) + .await? + } + }; + + if let Some(replace_table_info) = target_replace_info { + let stream_ctx = + StreamContext::from_protobuf(replace_table_info.fragment_graph.get_ctx().unwrap()); + + let ReplaceTableInfo { + mut streaming_job, + fragment_graph, + .. + } = replace_table_info; + + let sink_id = if let ObjectType::Sink = object_type { + object_id as _ + } else { + panic!("additional replace table event only occurs when dropping sink into table") + }; + + let fragment_graph = + StreamFragmentGraph::new(&self.env, fragment_graph, &streaming_job).await?; + streaming_job.set_table_fragment_id(fragment_graph.table_fragment_id()); + streaming_job.set_dml_fragment_id(fragment_graph.dml_fragment_id()); + let streaming_job = streaming_job; + + let table = streaming_job.table().unwrap(); + + tracing::debug!(id = streaming_job.id(), "replacing table for dropped sink"); + let dummy_id = mgr + .catalog_controller + .create_job_catalog_for_replace( + &streaming_job, + &stream_ctx, + table.get_version()?, + &fragment_graph.default_parallelism(), + ) + .await? as u32; + + let (ctx, table_fragments) = self + .inject_replace_table_job_for_table_sink( + dummy_id, + &self.metadata_manager, + stream_ctx, + None, + None, + Some(sink_id), + &streaming_job, + fragment_graph, + ) + .await?; + + let result: MetaResult> = try { + let merge_updates = ctx.merge_updates.clone(); + + mgr.catalog_controller + .prepare_streaming_job(table_fragments.to_protobuf(), &streaming_job, true) + .await?; + + self.stream_manager + .replace_table(table_fragments, ctx) + .await?; + + merge_updates + }; + + version = match result { + Ok(merge_updates) => { + let version = mgr + .catalog_controller + .finish_replace_streaming_job( + dummy_id as _, + streaming_job, + merge_updates, + None, + None, + Some(sink_id), + ) + .await?; + Ok(version) + } + Err(err) => { + tracing::error!(id = object_id, error = ?err.as_report(), "failed to replace table"); + let _ = mgr + .catalog_controller + .try_abort_replacing_streaming_job(dummy_id as _) + .await + .inspect_err(|err| { + tracing::error!(id = object_id, error = ?err.as_report(), "failed to abort replacing table"); + }); + Err(err) + } + }?; + } + + let ReleaseContext { + state_table_ids, + source_ids, + connections, + source_fragments, + removed_actors, + } = release_ctx; + + // delete vpc endpoints. + for conn in connections { + let _ = self + .delete_vpc_endpoint_v2(conn.into_inner()) + .await + .inspect_err(|err| { + tracing::warn!(err = ?err.as_report(), "failed to delete vpc endpoint"); + }); + } + + // unregister sources. + self.source_manager + .unregister_sources(source_ids.into_iter().map(|id| id as _).collect()) + .await; + + // unregister fragments and actors from source manager. + self.source_manager + .drop_source_fragments_v2( + source_fragments + .into_iter() + .map(|(source_id, fragments)| { + ( + source_id as u32, + fragments.into_iter().map(|id| id as u32).collect(), + ) + }) + .collect(), + removed_actors.iter().map(|id| *id as _).collect(), + ) + .await; + + // drop streaming jobs. + self.stream_manager + .drop_streaming_jobs_v2( + removed_actors.into_iter().map(|id| id as _).collect(), + state_table_ids.into_iter().map(|id| id as _).collect(), + ) + .await; + + Ok(version) + } + + /// This is used for `ALTER TABLE ADD/DROP COLUMN`. + pub async fn replace_table_v2( + &self, + mut streaming_job: StreamingJob, + fragment_graph: StreamFragmentGraphProto, + table_col_index_mapping: Option, + ) -> MetaResult { + let mgr = self.metadata_manager.as_v2_ref(); + let job_id = streaming_job.id(); + + let _reschedule_job_lock = self.stream_manager.reschedule_lock_read_guard().await; + let ctx = StreamContext::from_protobuf(fragment_graph.get_ctx().unwrap()); + + // 1. build fragment graph. + let fragment_graph = + StreamFragmentGraph::new(&self.env, fragment_graph, &streaming_job).await?; + streaming_job.set_table_fragment_id(fragment_graph.table_fragment_id()); + streaming_job.set_dml_fragment_id(fragment_graph.dml_fragment_id()); + let streaming_job = streaming_job; + + let StreamingJob::Table(_, table, ..) = &streaming_job else { + unreachable!("unexpected job: {streaming_job:?}") + }; + let dummy_id = mgr + .catalog_controller + .create_job_catalog_for_replace( + &streaming_job, + &ctx, + table.get_version()?, + &fragment_graph.default_parallelism(), + ) + .await?; + + tracing::debug!(id = streaming_job.id(), "building replace streaming job"); + let result: MetaResult> = try { + let (ctx, table_fragments) = self + .build_replace_table( + ctx, + &streaming_job, + fragment_graph, + table_col_index_mapping.clone(), + dummy_id as _, + ) + .await?; + let merge_updates = ctx.merge_updates.clone(); + + mgr.catalog_controller + .prepare_streaming_job(table_fragments.to_protobuf(), &streaming_job, true) + .await?; + + self.stream_manager + .replace_table(table_fragments, ctx) + .await?; + merge_updates + }; + + match result { + Ok(merge_updates) => { + let version = mgr + .catalog_controller + .finish_replace_streaming_job( + dummy_id, + streaming_job, + merge_updates, + table_col_index_mapping, + None, + None, + ) + .await?; + Ok(version) + } + Err(err) => { + tracing::error!(id = job_id, error = ?err.as_report(), "failed to replace table"); + let _ = mgr + .catalog_controller + .try_abort_replacing_streaming_job(dummy_id) + .await.inspect_err(|err| { + tracing::error!(id = job_id, error = ?err.as_report(), "failed to abort replacing table"); + }); + Err(err) + } + } + } } diff --git a/src/meta/src/rpc/election/etcd.rs b/src/meta/src/rpc/election/etcd.rs index 3cfb0bea6eba2..96b16f537356e 100644 --- a/src/meta/src/rpc/election/etcd.rs +++ b/src/meta/src/rpc/election/etcd.rs @@ -18,6 +18,7 @@ use std::time::Duration; use etcd_client::{ConnectOptions, Error, GetOptions, LeaderKey, ResignOptions}; use risingwave_common::bail; +use thiserror_ext::AsReport; use tokio::sync::watch::Receiver; use tokio::sync::{oneshot, watch}; use tokio::time; @@ -118,9 +119,9 @@ impl ElectionClient for EtcdElectionClient { Ok(resp) => resp, Err(e) => { tracing::warn!( - "create lease keeper for {} failed {}", + error = %e.as_report(), + "create lease keeper for {} failed", lease_id, - e.to_string() ); keep_alive_fail_tx.send(()).unwrap(); return; @@ -148,7 +149,7 @@ impl ElectionClient for EtcdElectionClient { _ = ticker.tick(), if !keep_alive_sending => { if let Err(err) = keeper.keep_alive().await { - tracing::debug!("keep alive for lease {} failed {}", lease_id, err); + tracing::debug!(error = %err.as_report(), "keep alive for lease {} failed", lease_id); continue } @@ -179,7 +180,7 @@ impl ElectionClient for EtcdElectionClient { continue; } Err(e) => { - tracing::error!("lease keeper failed {}", e.to_string()); + tracing::error!(error = %e.as_report(), "lease keeper failed"); continue; } }; @@ -264,7 +265,7 @@ impl ElectionClient for EtcdElectionClient { } } Some(Err(e)) => { - tracing::warn!("error {} received from leader observe stream", e.to_string()); + tracing::warn!(error = %e.as_report(), "error received from leader observe stream"); continue } } diff --git a/src/meta/src/rpc/election/sql.rs b/src/meta/src/rpc/election/sql.rs index b076f542a3700..65c3ad613dde0 100644 --- a/src/meta/src/rpc/election/sql.rs +++ b/src/meta/src/rpc/election/sql.rs @@ -20,6 +20,7 @@ use sea_orm::{ ConnectionTrait, DatabaseBackend, DatabaseConnection, FromQueryResult, Statement, TransactionTrait, Value, }; +use thiserror_ext::AsReport; use tokio::sync::watch; use tokio::sync::watch::Receiver; use tokio::time; @@ -614,7 +615,7 @@ where .update_heartbeat(META_ELECTION_KEY, id.as_str()) .await { - tracing::debug!("keep alive for member {} failed {}", id, e); + tracing::debug!(error = %e.as_report(), "keep alive for member {} failed", id); continue } } @@ -669,7 +670,7 @@ where if is_leader { tracing::info!("leader {} resigning", self.id); if let Err(e) = self.driver.resign(META_ELECTION_KEY, self.id.as_str()).await { - tracing::warn!("resign failed {}", e); + tracing::warn!(error = %e.as_report(), "resign failed"); } } diff --git a/src/meta/src/rpc/metrics.rs b/src/meta/src/rpc/metrics.rs index 086d8656839f2..d2dca20f98edf 100644 --- a/src/meta/src/rpc/metrics.rs +++ b/src/meta/src/rpc/metrics.rs @@ -518,7 +518,7 @@ impl MetaMetrics { let actor_info = register_int_gauge_vec_with_registry!( "actor_info", - "Mapping from actor id to (fragment id, compute node", + "Mapping from actor id to (fragment id, compute node)", &["actor_id", "fragment_id", "compute_node"], registry ) diff --git a/src/meta/src/stream/scale.rs b/src/meta/src/stream/scale.rs index fc0a7ef55b8f1..3b65c73d059cc 100644 --- a/src/meta/src/stream/scale.rs +++ b/src/meta/src/stream/scale.rs @@ -30,29 +30,30 @@ use risingwave_common::buffer::{Bitmap, BitmapBuilder}; use risingwave_common::catalog::TableId; use risingwave_common::hash::{ActorMapping, ParallelUnitId, VirtualNode}; use risingwave_common::util::iter_util::ZipEqDebug; -use risingwave_pb::common::{ActorInfo, ParallelUnit, WorkerNode}; +use risingwave_meta_model_v2::StreamingParallelism; +use risingwave_pb::common::{ + ActorInfo, Buffer, ParallelUnit, ParallelUnitMapping, WorkerNode, WorkerType, +}; use risingwave_pb::meta::get_reschedule_plan_request::{Policy, StableResizePolicy}; use risingwave_pb::meta::subscribe_response::{Info, Operation}; use risingwave_pb::meta::table_fragments::actor_status::ActorState; -use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType; -use risingwave_pb::meta::table_fragments::{self, ActorStatus, Fragment}; +use risingwave_pb::meta::table_fragments::fragment::{ + FragmentDistributionType, PbFragmentDistributionType, +}; +use risingwave_pb::meta::table_fragments::{self, ActorStatus, PbFragment, State}; use risingwave_pb::meta::FragmentParallelUnitMappings; use risingwave_pb::stream_plan::stream_node::NodeBody; -use risingwave_pb::stream_plan::{DispatcherType, FragmentTypeFlag, StreamActor, StreamNode}; -use risingwave_pb::stream_service::{ - BroadcastActorInfoTableRequest, BuildActorsRequest, UpdateActorsRequest, +use risingwave_pb::stream_plan::{ + Dispatcher, DispatcherType, FragmentTypeFlag, PbStreamActor, StreamNode, }; -use tokio::sync::oneshot; +use thiserror_ext::AsReport; use tokio::sync::oneshot::Receiver; +use tokio::sync::{oneshot, RwLock, RwLockReadGuard, RwLockWriteGuard}; use tokio::task::JoinHandle; -use tokio::time::MissedTickBehavior; -use uuid::Uuid; +use tokio::time::{Instant, MissedTickBehavior}; -use crate::barrier::{Command, Reschedule}; -use crate::manager::{ - ClusterManagerRef, FragmentManagerRef, IdCategory, LocalNotification, MetaSrvEnv, - MetadataManager, WorkerId, -}; +use crate::barrier::{Command, Reschedule, StreamRpcManager}; +use crate::manager::{IdCategory, LocalNotification, MetaSrvEnv, MetadataManager, WorkerId}; use crate::model::{ActorId, DispatcherId, FragmentId, TableFragments, TableParallelism}; use crate::serving::{ to_deleted_fragment_parallel_unit_mapping, to_fragment_parallel_unit_mapping, @@ -60,9 +61,9 @@ use crate::serving::{ }; use crate::storage::{MetaStore, MetaStoreError, MetaStoreRef, Transaction, DEFAULT_COLUMN_FAMILY}; use crate::stream::{GlobalStreamManager, SourceManagerRef}; -use crate::{MetaError, MetaResult}; +use crate::{model, MetaError, MetaResult}; -#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] +#[derive(Default, Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd)] pub struct TableRevision(u64); const TABLE_REVISION_KEY: &[u8] = b"table_revision"; @@ -110,15 +111,85 @@ pub struct ParallelUnitReschedule { pub removed_parallel_units: BTreeSet, } +pub struct CustomFragmentInfo { + pub fragment_id: u32, + pub fragment_type_mask: u32, + pub distribution_type: PbFragmentDistributionType, + pub vnode_mapping: Option, + pub state_table_ids: Vec, + pub upstream_fragment_ids: Vec, + pub actor_template: PbStreamActor, + pub actors: Vec, +} + +#[derive(Default)] +pub struct CustomActorInfo { + pub actor_id: u32, + pub fragment_id: u32, + pub dispatcher: Vec, + pub upstream_actor_id: Vec, + pub vnode_bitmap: Option, +} + +impl From<&PbStreamActor> for CustomActorInfo { + fn from( + PbStreamActor { + actor_id, + fragment_id, + dispatcher, + upstream_actor_id, + vnode_bitmap, + .. + }: &PbStreamActor, + ) -> Self { + CustomActorInfo { + actor_id: *actor_id, + fragment_id: *fragment_id, + dispatcher: dispatcher.clone(), + upstream_actor_id: upstream_actor_id.clone(), + vnode_bitmap: vnode_bitmap.clone(), + } + } +} + +impl From<&PbFragment> for CustomFragmentInfo { + fn from(fragment: &PbFragment) -> Self { + CustomFragmentInfo { + fragment_id: fragment.fragment_id, + fragment_type_mask: fragment.fragment_type_mask, + distribution_type: fragment.distribution_type(), + vnode_mapping: fragment.vnode_mapping.clone(), + state_table_ids: fragment.state_table_ids.clone(), + upstream_fragment_ids: fragment.upstream_fragment_ids.clone(), + actor_template: fragment + .actors + .first() + .cloned() + .expect("no actor in fragment"), + actors: fragment.actors.iter().map(CustomActorInfo::from).collect(), + } + } +} + +impl CustomFragmentInfo { + pub fn get_fragment_type_mask(&self) -> u32 { + self.fragment_type_mask + } + + pub fn distribution_type(&self) -> FragmentDistributionType { + self.distribution_type + } +} + pub struct RescheduleContext { /// Index used to map `ParallelUnitId` to `WorkerId` parallel_unit_id_to_worker_id: BTreeMap, /// Meta information for all Actors - actor_map: HashMap, + actor_map: HashMap, /// Status of all Actors, used to find the location of the `Actor` actor_status: BTreeMap, /// Meta information of all `Fragment`, used to find the `Fragment`'s `Actor` - fragment_map: HashMap, + fragment_map: HashMap, /// Indexes for all `Worker`s worker_nodes: HashMap, /// Index of all `Actor` upstreams, specific to `Dispatcher` @@ -185,10 +256,15 @@ impl RescheduleContext { /// /// The return value is the bitmap distribution after scaling, which covers all virtual node indexes pub fn rebalance_actor_vnode( - actors: &[StreamActor], + actors: &[CustomActorInfo], actors_to_remove: &BTreeSet, actors_to_create: &BTreeSet, ) -> HashMap { + let actor_ids: BTreeSet<_> = actors.iter().map(|actor| actor.actor_id).collect(); + + assert_eq!(actors_to_remove.difference(&actor_ids).count(), 0); + assert_eq!(actors_to_create.intersection(&actor_ids).count(), 0); + assert!(actors.len() >= actors_to_remove.len()); let target_actor_count = actors.len() - actors_to_remove.len() + actors_to_create.len(); @@ -368,29 +444,30 @@ pub struct RescheduleOptions { pub type ScaleControllerRef = Arc; pub struct ScaleController { - pub(super) fragment_manager: FragmentManagerRef, - - pub cluster_manager: ClusterManagerRef, + pub metadata_manager: MetadataManager, pub source_manager: SourceManagerRef, + pub stream_rpc_manager: StreamRpcManager, + pub env: MetaSrvEnv, + + pub reschedule_lock: RwLock<()>, } impl ScaleController { pub fn new( metadata_manager: &MetadataManager, source_manager: SourceManagerRef, + stream_rpc_manager: StreamRpcManager, env: MetaSrvEnv, ) -> Self { - match metadata_manager { - MetadataManager::V1(mgr) => Self { - fragment_manager: mgr.fragment_manager.clone(), - cluster_manager: mgr.cluster_manager.clone(), - source_manager, - env, - }, - MetadataManager::V2(_) => unimplemented!("support v2 in scale controller"), + Self { + stream_rpc_manager, + metadata_manager: metadata_manager.clone(), + source_manager, + env, + reschedule_lock: RwLock::new(()), } } @@ -401,11 +478,10 @@ impl ScaleController { options: RescheduleOptions, table_parallelisms: Option<&mut HashMap>, ) -> MetaResult { - // Index worker node, used to create actor let worker_nodes: HashMap = self - .cluster_manager + .metadata_manager .list_active_streaming_compute_nodes() - .await + .await? .into_iter() .map(|worker_node| (worker_node.id, worker_node)) .collect(); @@ -463,14 +539,30 @@ impl ScaleController { let mut actor_status = BTreeMap::new(); let mut fragment_state = HashMap::new(); let mut fragment_to_table = HashMap::new(); - for table_fragments in self.fragment_manager.list_table_fragments().await { + + // We are reusing code for the metadata manager of both V1 and V2, which will be deprecated in the future. + fn fulfill_index_by_table_fragments_ref( + actor_map: &mut HashMap, + fragment_map: &mut HashMap, + actor_status: &mut BTreeMap, + fragment_state: &mut HashMap, + fragment_to_table: &mut HashMap, + table_fragments: &TableFragments, + ) { fragment_state.extend( table_fragments .fragment_ids() .map(|f| (f, table_fragments.state())), ); - fragment_map.extend(table_fragments.fragments.clone()); - actor_map.extend(table_fragments.actor_map()); + + for (fragment_id, fragment) in &table_fragments.fragments { + for actor in &fragment.actors { + actor_map.insert(actor.actor_id, CustomActorInfo::from(actor)); + } + + fragment_map.insert(*fragment_id, CustomFragmentInfo::from(fragment)); + } + actor_status.extend(table_fragments.actor_status.clone()); fragment_to_table.extend( @@ -480,6 +572,37 @@ impl ScaleController { ); } + match &self.metadata_manager { + MetadataManager::V1(mgr) => { + let guard = mgr.fragment_manager.get_fragment_read_guard().await; + + for table_fragments in guard.table_fragments().values() { + fulfill_index_by_table_fragments_ref( + &mut actor_map, + &mut fragment_map, + &mut actor_status, + &mut fragment_state, + &mut fragment_to_table, + table_fragments, + ); + } + } + MetadataManager::V2(_) => { + let all_table_fragments = self.list_all_table_fragments().await?; + + for table_fragments in &all_table_fragments { + fulfill_index_by_table_fragments_ref( + &mut actor_map, + &mut fragment_map, + &mut actor_status, + &mut fragment_state, + &mut fragment_to_table, + table_fragments, + ); + } + } + }; + // NoShuffle relation index let mut no_shuffle_source_fragment_ids = HashSet::new(); let mut no_shuffle_target_fragment_ids = HashSet::new(); @@ -605,7 +728,7 @@ impl ScaleController { } if (fragment.get_fragment_type_mask() & FragmentTypeFlag::Source as u32) != 0 { - let stream_node = fragment.actors.first().unwrap().get_nodes().unwrap(); + let stream_node = fragment.actor_template.nodes.as_ref().unwrap(); if TableFragments::find_stream_source(stream_node).is_some() { stream_source_fragment_ids.insert(*fragment_id); } @@ -694,52 +817,35 @@ impl ScaleController { async fn create_actors_on_compute_node( &self, worker_nodes: &HashMap, - actor_infos_to_broadcast: BTreeMap, - node_actors_to_create: HashMap>, - broadcast_worker_ids: HashSet, + actor_infos_to_broadcast: BTreeMap, + node_actors_to_create: HashMap>, + broadcast_worker_ids: HashSet, ) -> MetaResult<()> { - for worker_id in &broadcast_worker_ids { - let node = worker_nodes.get(worker_id).unwrap(); - let client = self.env.stream_client_pool().get(node).await?; - - let actor_infos_to_broadcast = actor_infos_to_broadcast.values().cloned().collect(); - - client - .to_owned() - .broadcast_actor_info_table(BroadcastActorInfoTableRequest { - info: actor_infos_to_broadcast, - }) - .await?; - } - - for (node_id, stream_actors) in &node_actors_to_create { - let node = worker_nodes.get(node_id).unwrap(); - let client = self.env.stream_client_pool().get(node).await?; - let request_id = Uuid::new_v4().to_string(); - let request = UpdateActorsRequest { - request_id, - actors: stream_actors.clone(), - }; - - client.to_owned().update_actors(request).await?; - } - - for (node_id, stream_actors) in node_actors_to_create { - let node = worker_nodes.get(&node_id).unwrap(); - let client = self.env.stream_client_pool().get(node).await?; - let request_id = Uuid::new_v4().to_string(); + self.stream_rpc_manager + .broadcast_update_actor_info( + worker_nodes, + broadcast_worker_ids.into_iter(), + actor_infos_to_broadcast.values().cloned(), + node_actors_to_create.clone().into_iter(), + ) + .await?; - client - .to_owned() - .build_actors(BuildActorsRequest { - request_id, - actor_id: stream_actors - .iter() - .map(|stream_actor| stream_actor.actor_id) - .collect(), - }) - .await?; - } + self.stream_rpc_manager + .build_actors( + worker_nodes, + node_actors_to_create + .iter() + .map(|(node_id, stream_actors)| { + ( + *node_id, + stream_actors + .iter() + .map(|stream_actor| stream_actor.actor_id) + .collect_vec(), + ) + }), + ) + .await?; Ok(()) } @@ -977,7 +1083,7 @@ impl ScaleController { for (actor_to_create, sample_actor) in actors_to_create .iter() - .zip_eq_debug(repeat(fragment.actors.first().unwrap()).take(actors_to_create.len())) + .zip_eq_debug(repeat(&fragment.actor_template).take(actors_to_create.len())) { let new_actor_id = actor_to_create.0; let mut new_actor = sample_actor.clone(); @@ -1129,12 +1235,27 @@ impl ScaleController { HashMap::with_capacity(reschedules.len()); for (fragment_id, _) in reschedules { - let actors_to_create = fragment_actors_to_create + let mut actors_to_create: HashMap<_, Vec<_>> = HashMap::new(); + let fragment_type_mask = ctx + .fragment_map .get(&fragment_id) - .cloned() - .unwrap_or_default() - .into_keys() - .collect(); + .unwrap() + .fragment_type_mask; + let injectable = TableFragments::is_injectable(fragment_type_mask); + + if let Some(actor_pu_maps) = fragment_actors_to_create.get(&fragment_id).cloned() { + for (actor_id, parallel_unit_id) in actor_pu_maps { + let worker_id = ctx + .parallel_unit_id_to_worker_id + .get(¶llel_unit_id) + .with_context(|| format!("parallel unit {} not found", parallel_unit_id))?; + actors_to_create + .entry(*worker_id) + .or_default() + .push(actor_id); + } + } + let actors_to_remove = fragment_actors_to_remove .get(&fragment_id) .cloned() @@ -1274,6 +1395,8 @@ impl ScaleController { upstream_dispatcher_mapping, downstream_fragment_ids, actor_splits, + injectable, + newly_created_actors: vec![], }, ); } @@ -1306,8 +1429,13 @@ impl ScaleController { fragment_created_actors.insert(*fragment_id, created_actors); } + for (fragment_id, to_create) in &fragment_created_actors { + let reschedule = reschedule_fragment.get_mut(fragment_id).unwrap(); + reschedule.newly_created_actors = to_create.values().cloned().collect(); + } + let applied_reschedules = self - .fragment_manager + .metadata_manager .pre_apply_reschedules(fragment_created_actors) .await; @@ -1361,11 +1489,19 @@ impl ScaleController { } for created_parallel_unit_id in added_parallel_units { - let id = self - .env - .id_gen_manager() - .generate::<{ IdCategory::Actor }>() - .await? as ActorId; + let id = match self.env.sql_id_gen_manager_ref() { + None => { + self.env + .id_gen_manager() + .generate::<{ IdCategory::Actor }>() + .await? as ActorId + } + Some(id_gen) => { + let id = id_gen.generate_interval::<{ IdCategory::Actor }>(1); + + id as ActorId + } + }; actors_to_create.insert(id, *created_parallel_unit_id); } @@ -1391,7 +1527,7 @@ impl ScaleController { fragment_actor_bitmap: &HashMap>, no_shuffle_upstream_actor_map: &HashMap>, no_shuffle_downstream_actors_map: &HashMap>, - new_actor: &mut StreamActor, + new_actor: &mut PbStreamActor, ) -> MetaResult<()> { let fragment = &ctx.fragment_map.get(&new_actor.fragment_id).unwrap(); let mut applied_upstream_fragment_actor_ids = HashMap::new(); @@ -1535,50 +1671,22 @@ impl ScaleController { &self, reschedules: &HashMap, table_parallelism: &HashMap, - ) -> MetaResult>> { - let mut node_dropped_actors = HashMap::new(); - for table_fragments in self - .fragment_manager - .get_fragment_read_guard() - .await - .table_fragments() - .values() - { - for fragment_id in table_fragments.fragments.keys() { - if let Some(reschedule) = reschedules.get(fragment_id) { - for actor_id in &reschedule.removed_actors { - let node_id = table_fragments - .actor_status - .get(actor_id) - .unwrap() - .parallel_unit - .as_ref() - .unwrap() - .worker_node_id; - node_dropped_actors - .entry(node_id as WorkerId) - .or_insert(vec![]) - .push(*actor_id as ActorId); - } - } - } - } - + ) -> MetaResult<()> { // Update fragment info after rescheduling in meta store. - self.fragment_manager + self.metadata_manager .post_apply_reschedules(reschedules.clone(), table_parallelism.clone()) .await?; // Update serving fragment info after rescheduling in meta store. if !reschedules.is_empty() { let workers = self - .cluster_manager - .list_active_serving_compute_nodes() - .await; + .metadata_manager + .list_active_streaming_compute_nodes() + .await?; let streaming_parallelisms = self - .fragment_manager + .metadata_manager .running_fragment_parallelisms(Some(reschedules.keys().cloned().collect())) - .await; + .await?; let serving_vnode_mapping = Arc::new(ServingVnodeMapping::default()); let (upserted, failed) = serving_vnode_mapping.upsert(streaming_parallelisms, &workers); if !upserted.is_empty() { @@ -1632,7 +1740,24 @@ impl ScaleController { .await; } - Ok(node_dropped_actors) + Ok(()) + } + + // FIXME: should be removed + async fn list_all_table_fragments(&self) -> MetaResult> { + use crate::model::MetadataModel; + let all_table_fragments = match &self.metadata_manager { + MetadataManager::V1(mgr) => mgr.fragment_manager.list_table_fragments().await, + MetadataManager::V2(mgr) => mgr + .catalog_controller + .table_fragments() + .await? + .into_values() + .map(model::TableFragments::from_protobuf) + .collect(), + }; + + Ok(all_table_fragments) } pub async fn generate_table_resize_plan( @@ -1645,9 +1770,9 @@ impl ScaleController { } = policy; let workers = self - .cluster_manager + .metadata_manager .list_active_streaming_compute_nodes() - .await; + .await?; let unschedulable_worker_ids = Self::filter_unschedulable_workers(&workers); @@ -1676,60 +1801,144 @@ impl ScaleController { }) .collect::>(); - let all_table_fragments = self.fragment_manager.list_table_fragments().await; + // index for no shuffle relation + let mut no_shuffle_source_fragment_ids = HashSet::new(); + let mut no_shuffle_target_fragment_ids = HashSet::new(); - // FIXME: only need actor id and dispatcher info, avoid clone it. - let mut actor_map = HashMap::new(); + // index for fragment_id -> distribution_type + let mut fragment_distribution_map = HashMap::new(); + // index for actor -> parallel_unit let mut actor_status = HashMap::new(); - // FIXME: only need fragment distribution info, should avoid clone it. + // index for table_id -> [fragment_id] + let mut table_fragment_id_map = HashMap::new(); + // index for fragment_id -> [actor_id] + let mut fragment_actor_id_map = HashMap::new(); + + // internal helper func for building index + fn build_index( + no_shuffle_source_fragment_ids: &mut HashSet, + no_shuffle_target_fragment_ids: &mut HashSet, + fragment_distribution_map: &mut HashMap, + actor_status: &mut HashMap, + table_fragment_id_map: &mut HashMap>, + fragment_actor_id_map: &mut HashMap>, + table_fragments: &BTreeMap, + ) { + // This is only for assertion purposes and will be removed once the dispatcher_id is guaranteed to always correspond to the downstream fragment_id, + // such as through the foreign key constraints in the SQL backend. + let mut actor_fragment_id_map_for_check = HashMap::new(); + for table_fragments in table_fragments.values() { + for (fragment_id, fragment) in &table_fragments.fragments { + for actor in &fragment.actors { + debug_assert!(actor_fragment_id_map_for_check + .insert(actor.actor_id, *fragment_id) + .is_none()); + } + } + } - let mut table_fragment_map = HashMap::new(); - for table_fragments in all_table_fragments { - let table_id = table_fragments.table_id().table_id; - for (fragment_id, fragment) in table_fragments.fragments { - fragment - .actors - .iter() - .map(|actor| (actor.actor_id, actor)) - .for_each(|(id, actor)| { - actor_map.insert(id as ActorId, actor.clone()); - }); + for (table_id, table_fragments) in table_fragments { + for (fragment_id, fragment) in &table_fragments.fragments { + for actor in &fragment.actors { + fragment_actor_id_map + .entry(*fragment_id) + .or_default() + .insert(actor.actor_id); + + for dispatcher in &actor.dispatcher { + if dispatcher.r#type() == DispatcherType::NoShuffle { + no_shuffle_source_fragment_ids + .insert(actor.fragment_id as FragmentId); + + let downstream_actor_id = + dispatcher.downstream_actor_id.iter().exactly_one().expect( + "no shuffle should have exactly one downstream actor id", + ); + + if let Some(downstream_fragment_id) = + actor_fragment_id_map_for_check.get(downstream_actor_id) + { + // dispatcher_id of dispatcher should be exactly same as downstream fragment id + // but we need to check it to make sure + debug_assert_eq!( + *downstream_fragment_id, + dispatcher.dispatcher_id as FragmentId + ); + } else { + tracing::warn!( + "downstream actor id {} not found in fragment_actor_id_map", + downstream_actor_id + ); + } - table_fragment_map - .entry(table_id) - .or_insert(HashMap::new()) - .insert(fragment_id, fragment); - } + no_shuffle_target_fragment_ids + .insert(dispatcher.dispatcher_id as FragmentId); + } + } + } - actor_status.extend(table_fragments.actor_status); - } + fragment_distribution_map.insert(*fragment_id, fragment.distribution_type()); - let mut no_shuffle_source_fragment_ids = HashSet::new(); - let mut no_shuffle_target_fragment_ids = HashSet::new(); + table_fragment_id_map + .entry(table_id.table_id()) + .or_default() + .insert(*fragment_id); + } - Self::build_no_shuffle_relation_index( - &actor_map, - &mut no_shuffle_source_fragment_ids, - &mut no_shuffle_target_fragment_ids, - ); + actor_status.extend(table_fragments.actor_status.clone()); + } + } + + match &self.metadata_manager { + MetadataManager::V1(mgr) => { + let guard = mgr.fragment_manager.get_fragment_read_guard().await; + build_index( + &mut no_shuffle_source_fragment_ids, + &mut no_shuffle_target_fragment_ids, + &mut fragment_distribution_map, + &mut actor_status, + &mut table_fragment_id_map, + &mut fragment_actor_id_map, + guard.table_fragments(), + ); + } + MetadataManager::V2(_) => { + let all_table_fragments = self.list_all_table_fragments().await?; + let all_table_fragments = all_table_fragments + .into_iter() + .map(|table_fragments| (table_fragments.table_id(), table_fragments)) + .collect::>(); + + build_index( + &mut no_shuffle_source_fragment_ids, + &mut no_shuffle_target_fragment_ids, + &mut fragment_distribution_map, + &mut actor_status, + &mut table_fragment_id_map, + &mut fragment_actor_id_map, + &all_table_fragments, + ); + } + } let mut target_plan = HashMap::new(); for (table_id, parallelism) in table_parallelisms { - let fragment_map = table_fragment_map.remove(&table_id).unwrap(); + let fragment_map = table_fragment_id_map.remove(&table_id).unwrap(); - for (fragment_id, fragment) in fragment_map { + for fragment_id in fragment_map { // Currently, all of our NO_SHUFFLE relation propagations are only transmitted from upstream to downstream. if no_shuffle_target_fragment_ids.contains(&fragment_id) { continue; } - let fragment_parallel_unit_ids: BTreeSet<_> = fragment - .actors + let fragment_parallel_unit_ids: BTreeSet = fragment_actor_id_map + .get(&fragment_id) + .unwrap() .iter() - .map(|actor| { + .map(|actor_id| { actor_status - .get(&actor.actor_id) + .get(actor_id) .and_then(|status| status.parallel_unit.clone()) .unwrap() .id as ParallelUnitId @@ -1746,7 +1955,7 @@ impl ScaleController { ); } - match fragment.get_distribution_type().unwrap() { + match fragment_distribution_map.get(&fragment_id).unwrap() { FragmentDistributionType::Unspecified => unreachable!(), FragmentDistributionType::Single => { let single_parallel_unit_id = @@ -1764,13 +1973,8 @@ impl ScaleController { .flatten() .cloned() .exactly_one() - .map_err(|e| { - anyhow!( - "Cannot find a single target ParallelUnit for fragment {}: {}", - fragment_id, - e - ) - })?; + .ok() + .with_context(|| format!("Cannot find a single target ParallelUnit for fragment {fragment_id}"))?; target_plan.insert( fragment_id, @@ -1783,7 +1987,7 @@ impl ScaleController { ); } FragmentDistributionType::Hash => match parallelism { - TableParallelism::Auto => { + TableParallelism::Adaptive => { target_plan.insert( fragment_id, Self::diff_parallel_unit_change( @@ -1841,9 +2045,9 @@ impl ScaleController { let mut target_plan = HashMap::with_capacity(fragment_worker_changes.len()); let workers = self - .cluster_manager + .metadata_manager .list_active_streaming_compute_nodes() - .await; + .await?; let unschedulable_worker_ids = Self::filter_unschedulable_workers(&workers); @@ -1869,8 +2073,6 @@ impl ScaleController { }) .collect::>(); - let all_table_fragments = self.fragment_manager.list_table_fragments().await; - // FIXME: only need actor id and dispatcher info, avoid clone it. let mut actor_map = HashMap::new(); let mut actor_status = HashMap::new(); @@ -1878,24 +2080,56 @@ impl ScaleController { let mut fragment_map = HashMap::new(); let mut fragment_parallelism = HashMap::new(); - for table_fragments in all_table_fragments { - for (fragment_id, fragment) in table_fragments.fragments { - fragment - .actors - .iter() - .map(|actor| (actor.actor_id, actor)) - .for_each(|(id, actor)| { - actor_map.insert(id as ActorId, actor.clone()); - }); + // We are reusing code for the metadata manager of both V1 and V2, which will be deprecated in the future. + fn fulfill_index_by_table_fragments_ref( + actor_map: &mut HashMap, + actor_status: &mut HashMap, + fragment_map: &mut HashMap, + fragment_parallelism: &mut HashMap, + table_fragments: &TableFragments, + ) { + for (fragment_id, fragment) in &table_fragments.fragments { + for actor in &fragment.actors { + actor_map.insert(actor.actor_id, CustomActorInfo::from(actor)); + } - fragment_map.insert(fragment_id, fragment); + fragment_map.insert(*fragment_id, CustomFragmentInfo::from(fragment)); - fragment_parallelism.insert(fragment_id, table_fragments.assigned_parallelism); + fragment_parallelism.insert(*fragment_id, table_fragments.assigned_parallelism); } - actor_status.extend(table_fragments.actor_status); + actor_status.extend(table_fragments.actor_status.clone()); } + match &self.metadata_manager { + MetadataManager::V1(mgr) => { + let guard = mgr.fragment_manager.get_fragment_read_guard().await; + + for table_fragments in guard.table_fragments().values() { + fulfill_index_by_table_fragments_ref( + &mut actor_map, + &mut actor_status, + &mut fragment_map, + &mut fragment_parallelism, + table_fragments, + ); + } + } + MetadataManager::V2(_) => { + let all_table_fragments = self.list_all_table_fragments().await?; + + for table_fragments in &all_table_fragments { + fulfill_index_by_table_fragments_ref( + &mut actor_map, + &mut actor_status, + &mut fragment_map, + &mut fragment_parallelism, + table_fragments, + ); + } + } + }; + let mut no_shuffle_source_fragment_ids = HashSet::new(); let mut no_shuffle_target_fragment_ids = HashSet::new(); @@ -1950,7 +2184,7 @@ impl ScaleController { }, ) in fragment_worker_changes { - let fragment = match fragment_map.get(&fragment_id).cloned() { + let fragment = match fragment_map.get(&fragment_id) { None => bail!("Fragment id {} not found", fragment_id), Some(fragment) => fragment, }; @@ -2038,7 +2272,7 @@ impl ScaleController { // then we re-add the limited parallel units from the limited workers target_parallel_unit_ids.extend(limited_worker_parallel_unit_ids.into_iter()); } - match fragment.get_distribution_type().unwrap() { + match fragment.distribution_type() { FragmentDistributionType::Unspecified => unreachable!(), FragmentDistributionType::Single => { let single_parallel_unit_id = @@ -2190,7 +2424,7 @@ impl ScaleController { } pub fn build_no_shuffle_relation_index( - actor_map: &HashMap, + actor_map: &HashMap, no_shuffle_source_fragment_ids: &mut HashSet, no_shuffle_target_fragment_ids: &mut HashSet, ) { @@ -2218,7 +2452,7 @@ impl ScaleController { } pub fn build_fragment_dispatcher_index( - actor_map: &HashMap, + actor_map: &HashMap, fragment_dispatcher_map: &mut HashMap>, ) { for actor in actor_map.values() { @@ -2240,7 +2474,7 @@ impl ScaleController { pub fn resolve_no_shuffle_upstream_tables( fragment_ids: HashSet, - fragment_map: &HashMap, + fragment_map: &HashMap, no_shuffle_source_fragment_ids: &HashSet, no_shuffle_target_fragment_ids: &HashSet, fragment_to_table: &HashMap, @@ -2310,7 +2544,7 @@ impl ScaleController { pub fn resolve_no_shuffle_upstream_fragments( reschedule: &mut HashMap, - fragment_map: &HashMap, + fragment_map: &HashMap, no_shuffle_source_fragment_ids: &HashSet, no_shuffle_target_fragment_ids: &HashSet, ) -> MetaResult<()> @@ -2368,6 +2602,14 @@ pub struct TableResizePolicy { } impl GlobalStreamManager { + pub async fn reschedule_lock_read_guard(&self) -> RwLockReadGuard<'_, ()> { + self.scale_controller.reschedule_lock.read().await + } + + pub async fn reschedule_lock_write_guard(&self) -> RwLockWriteGuard<'_, ()> { + self.scale_controller.reschedule_lock.write().await + } + pub async fn reschedule_actors( &self, reschedules: HashMap, @@ -2395,104 +2637,206 @@ impl GlobalStreamManager { options: RescheduleOptions, table_parallelism: Option>, ) -> MetaResult<()> { - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support reschedule in v2"); - }; - let mut table_parallelism = table_parallelism; let (reschedule_fragment, applied_reschedules) = self .scale_controller - .as_ref() - .unwrap() .prepare_reschedule_command(reschedules, options, table_parallelism.as_mut()) .await?; - tracing::debug!("reschedule plan: {:#?}", reschedule_fragment); + tracing::debug!("reschedule plan: {:?}", reschedule_fragment); let command = Command::RescheduleFragment { reschedules: reschedule_fragment, table_parallelism: table_parallelism.unwrap_or_default(), }; - let fragment_manager_ref = mgr.fragment_manager.clone(); + match &self.metadata_manager { + MetadataManager::V1(mgr) => { + let fragment_manager_ref = mgr.fragment_manager.clone(); - revert_funcs.push(Box::pin(async move { - fragment_manager_ref - .cancel_apply_reschedules(applied_reschedules) - .await; - })); + revert_funcs.push(Box::pin(async move { + fragment_manager_ref + .cancel_apply_reschedules(applied_reschedules) + .await; + })); + } + MetadataManager::V2(_) => { + // meta model v2 does not need to revert + } + } + tracing::debug!("pausing tick lock in source manager"); let _source_pause_guard = self.source_manager.paused.lock().await; self.barrier_scheduler .run_config_change_command_with_pause(command) .await?; + tracing::info!("reschedule done"); + Ok(()) } - async fn trigger_parallelism_control(&self) -> MetaResult<()> { - let _reschedule_job_lock = self.reschedule_lock.write().await; + async fn trigger_parallelism_control(&self) -> MetaResult { + tracing::info!("trigger parallelism control"); + + let _reschedule_job_lock = self.reschedule_lock_write_guard().await; + + let (schedulable_worker_ids, table_parallelisms) = match &self.metadata_manager { + MetadataManager::V1(mgr) => { + let table_parallelisms: HashMap = { + let guard = mgr.fragment_manager.get_fragment_read_guard().await; - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support reschedule in v2"); + guard + .table_fragments() + .iter() + .filter(|&(_, table)| matches!(table.state(), State::Created)) + .map(|(table_id, table)| (table_id.table_id, table.assigned_parallelism)) + .collect() + }; + + let workers = mgr + .cluster_manager + .list_active_streaming_compute_nodes() + .await; + + let schedulable_worker_ids: BTreeSet<_> = workers + .iter() + .filter(|worker| { + !worker + .property + .as_ref() + .map(|p| p.is_unschedulable) + .unwrap_or(false) + }) + .map(|worker| worker.id) + .collect(); + + (schedulable_worker_ids, table_parallelisms) + } + MetadataManager::V2(mgr) => { + let table_parallelisms: HashMap<_, _> = { + let streaming_parallelisms = mgr + .catalog_controller + .get_all_created_streaming_parallelisms() + .await?; + + streaming_parallelisms + .into_iter() + .map(|(table_id, parallelism)| { + // no custom for sql backend + let table_parallelism = match parallelism { + StreamingParallelism::Adaptive => TableParallelism::Adaptive, + StreamingParallelism::Fixed(n) => TableParallelism::Fixed(n), + }; + + (table_id as u32, table_parallelism) + }) + .collect() + }; + + let workers = mgr + .cluster_controller + .list_active_streaming_workers() + .await?; + + let schedulable_worker_ids = workers + .iter() + .filter(|worker| { + !worker + .property + .as_ref() + .map(|p| p.is_unschedulable) + .unwrap_or(false) + }) + .map(|worker| worker.id) + .collect(); + + (schedulable_worker_ids, table_parallelisms) + } }; - let table_parallelisms = { - let guard = mgr.fragment_manager.get_fragment_read_guard().await; + if table_parallelisms.is_empty() { + tracing::info!("no streaming jobs for scaling, maybe an empty cluster"); + return Ok(false); + } - guard - .table_fragments() - .iter() - .map(|(table_id, table)| (table_id.table_id, table.assigned_parallelism)) - .collect() + let batch_size = match self.env.opts.parallelism_control_batch_size { + 0 => table_parallelisms.len(), + n => n, }; - let workers = mgr - .cluster_manager - .list_active_streaming_compute_nodes() - .await; + tracing::info!( + "total {} streaming jobs, batch size {}, schedulable worker ids: {:?}", + table_parallelisms.len(), + batch_size, + schedulable_worker_ids + ); - let schedulable_worker_ids = workers - .iter() - .filter(|worker| { - !worker - .property - .as_ref() - .map(|p| p.is_unschedulable) - .unwrap_or(false) - }) - .map(|worker| worker.id) + let batches: Vec<_> = table_parallelisms + .into_iter() + .chunks(batch_size) + .into_iter() + .map(|chunk| chunk.collect_vec()) .collect(); - let reschedules = self - .scale_controller - .as_ref() - .unwrap() - .generate_table_resize_plan(TableResizePolicy { - worker_ids: schedulable_worker_ids, - table_parallelisms, - }) - .await?; + let mut reschedules = None; + + for batch in batches { + let parallelisms: HashMap<_, _> = batch.into_iter().collect(); - if reschedules.is_empty() { - return Ok(()); + let plan = self + .scale_controller + .generate_table_resize_plan(TableResizePolicy { + worker_ids: schedulable_worker_ids.clone(), + table_parallelisms: parallelisms.clone(), + }) + .await?; + + if !plan.is_empty() { + tracing::info!( + "reschedule plan generated for streaming jobs {:?}", + parallelisms + ); + reschedules = Some(plan); + break; + } } + let Some(reschedules) = reschedules else { + tracing::info!("no reschedule plan generated"); + return Ok(false); + }; + self.reschedule_actors( reschedules, RescheduleOptions { - resolve_no_shuffle_upstream: true, + resolve_no_shuffle_upstream: false, }, None, ) .await?; - Ok(()) + Ok(true) } async fn run(&self, mut shutdown_rx: Receiver<()>) { + tracing::info!("starting automatic parallelism control monitor"); + + let check_period = + Duration::from_secs(self.env.opts.parallelism_control_trigger_period_sec); + + let mut ticker = tokio::time::interval_at( + Instant::now() + + Duration::from_secs(self.env.opts.parallelism_control_trigger_first_delay_sec), + check_period, + ); + ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); + + // waiting for first tick + ticker.tick().await; + let (local_notification_tx, mut local_notification_rx) = tokio::sync::mpsc::unbounded_channel(); @@ -2501,11 +2845,6 @@ impl GlobalStreamManager { .insert_local_sender(local_notification_tx) .await; - let check_period = Duration::from_secs(10); - let mut ticker = tokio::time::interval(check_period); - ticker.set_missed_tick_behavior(MissedTickBehavior::Skip); - ticker.reset(); - let worker_nodes = self .metadata_manager .list_active_streaming_compute_nodes() @@ -2517,7 +2856,7 @@ impl GlobalStreamManager { .map(|worker| (worker.id, worker)) .collect(); - let mut changed = true; + let mut should_trigger = false; loop { tokio::select! { @@ -2528,21 +2867,21 @@ impl GlobalStreamManager { break; } - _ = ticker.tick(), if changed => { + _ = ticker.tick(), if should_trigger => { let include_workers = worker_cache.keys().copied().collect_vec(); if include_workers.is_empty() { tracing::debug!("no available worker nodes"); - changed = false; + should_trigger = false; continue; } match self.trigger_parallelism_control().await { - Ok(_) => { - changed = false; + Ok(cont) => { + should_trigger = cont; } Err(e) => { - tracing::warn!(error = e.to_string(), "Failed to trigger scale out, waiting for next tick to retry after {}s", ticker.period().as_secs()); + tracing::warn!(error = %e.as_report(), "Failed to trigger scale out, waiting for next tick to retry after {}s", ticker.period().as_secs()); ticker.reset(); } } @@ -2553,13 +2892,26 @@ impl GlobalStreamManager { match notification { LocalNotification::WorkerNodeActivated(worker) => { + match (worker.get_type(), worker.property.as_ref()) { + (Ok(WorkerType::ComputeNode), Some(prop)) if prop.is_streaming => { + tracing::info!("worker {} activated notification received", worker.id); + } + _ => continue + } + let prev_worker = worker_cache.insert(worker.id, worker.clone()); - if let Some(prev_worker) = prev_worker && prev_worker.parallel_units != worker.parallel_units { - tracing::info!(worker = worker.id, "worker parallelism changed"); + match prev_worker { + Some(prev_worker) if prev_worker.parallel_units != worker.parallel_units => { + tracing::info!(worker = worker.id, "worker parallelism changed"); + should_trigger = true; + } + None => { + tracing::info!(worker = worker.id, "new worker joined"); + should_trigger = true; + } + _ => {} } - - changed = true; } // Since our logic for handling passive scale-in is within the barrier manager, diff --git a/src/meta/src/stream/source_manager.rs b/src/meta/src/stream/source_manager.rs index 9f99aa0b405d8..eb3d6b3205c4c 100644 --- a/src/meta/src/stream/source_manager.rs +++ b/src/meta/src/stream/source_manager.rs @@ -20,10 +20,11 @@ use std::ops::Deref; use std::sync::Arc; use std::time::Duration; -use anyhow::{anyhow, Context}; +use anyhow::Context; use risingwave_common::catalog::TableId; use risingwave_common::metrics::LabelGuardedIntGauge; use risingwave_connector::dispatch_source_prop; +use risingwave_connector::error::ConnectorResult; use risingwave_connector::source::{ ConnectorProperties, SourceEnumeratorContext, SourceEnumeratorInfo, SourceProperties, SplitEnumerator, SplitId, SplitImpl, SplitMetaData, @@ -31,6 +32,7 @@ use risingwave_connector::source::{ use risingwave_pb::catalog::Source; use risingwave_pb::source::{ConnectorSplit, ConnectorSplits}; use risingwave_rpc_client::ConnectorClient; +use thiserror_ext::AsReport; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tokio::sync::{oneshot, Mutex}; use tokio::task::JoinHandle; @@ -80,12 +82,12 @@ struct ConnectorSourceWorker { source_is_up: LabelGuardedIntGauge<2>, } -fn extract_prop_from_existing_source(source: &Source) -> anyhow::Result { +fn extract_prop_from_existing_source(source: &Source) -> ConnectorResult { let mut properties = ConnectorProperties::extract(source.with_properties.clone(), false)?; properties.init_from_pb_source(source); Ok(properties) } -fn extract_prop_from_new_source(source: &Source) -> anyhow::Result { +fn extract_prop_from_new_source(source: &Source) -> ConnectorResult { let mut properties = ConnectorProperties::extract(source.with_properties.clone(), true)?; properties.init_from_pb_source(source); Ok(properties) @@ -95,7 +97,7 @@ const DEFAULT_SOURCE_WORKER_TICK_INTERVAL: Duration = Duration::from_secs(30); impl ConnectorSourceWorker

{ /// Recreate the `SplitEnumerator` to establish a new connection to the external source service. - async fn refresh(&mut self) -> anyhow::Result<()> { + async fn refresh(&mut self) -> MetaResult<()> { let enumerator = P::SplitEnumerator::new( self.connector_properties.clone(), Arc::new(SourceEnumeratorContext { @@ -123,7 +125,7 @@ impl ConnectorSourceWorker

{ period: Duration, splits: Arc>, metrics: Arc, - ) -> anyhow::Result { + ) -> MetaResult { let enumerator = P::SplitEnumerator::new( connector_properties.clone(), Arc::new(SourceEnumeratorContext { @@ -172,11 +174,11 @@ impl ConnectorSourceWorker

{ _ = interval.tick() => { if self.fail_cnt > MAX_FAIL_CNT { if let Err(e) = self.refresh().await { - tracing::error!("error happened when refresh from connector source worker: {}", e.to_string()); + tracing::error!(error = %e.as_report(), "error happened when refresh from connector source worker"); } } if let Err(e) = self.tick().await { - tracing::error!("error happened when tick from connector source worker: {}", e.to_string()); + tracing::error!(error = %e.as_report(), "error happened when tick from connector source worker"); } } } @@ -289,7 +291,7 @@ impl SourceManagerCore { { Ok(actor_ids) => actor_ids, Err(err) => { - tracing::warn!("Failed to get the actor of the fragment {}, maybe the fragment doesn't exist anymore", err.to_string()); + tracing::warn!(error = %err.as_report(), "Failed to get the actor of the fragment, maybe the fragment doesn't exist anymore"); continue; } }; @@ -360,7 +362,7 @@ impl SourceManagerCore { fn drop_source_fragments( &mut self, source_fragments: HashMap>, - actor_splits: &HashSet, + removed_actors: &HashSet, ) { for (source_id, fragment_ids) in source_fragments { if let Entry::Occupied(mut entry) = self.source_fragments.entry(source_id) { @@ -379,7 +381,7 @@ impl SourceManagerCore { } } - for actor_id in actor_splits { + for actor_id in removed_actors { self.actor_splits.remove(actor_id); } } @@ -612,6 +614,15 @@ impl SourceManager { }) } + pub async fn drop_source_fragments_v2( + &self, + source_fragments: HashMap>, + removed_actors: HashSet, + ) { + let mut core = self.core.lock().await; + core.drop_source_fragments(source_fragments, &removed_actors); + } + /// For dropping MV. pub async fn drop_source_fragments(&self, table_fragments: &[TableFragments]) { let mut core = self.core.lock().await; @@ -701,7 +712,7 @@ impl SourceManager { let handle = core .managed_sources .get(&source_id) - .ok_or_else(|| anyhow!("could not found source {}", source_id))?; + .with_context(|| format!("could not find source {}", source_id))?; if handle.splits.lock().await.splits.is_none() { // force refresh source @@ -709,8 +720,11 @@ impl SourceManager { handle .sync_call_tx .send(tx) - .map_err(|e| anyhow!(e.to_string()))?; - rx.await.map_err(|e| anyhow!(e.to_string()))??; + .ok() + .context("failed to send sync call")?; + rx.await + .ok() + .context("failed to receive sync call response")??; } let splits = handle.discovered_splits().await.unwrap(); @@ -745,7 +759,7 @@ impl SourceManager { } /// register connector worker for source. - pub async fn register_source(&self, source: &Source) -> anyhow::Result<()> { + pub async fn register_source(&self, source: &Source) -> MetaResult<()> { let mut core = self.core.lock().await; if core.managed_sources.contains_key(&source.get_id()) { tracing::warn!("source {} already registered", source.get_id()); @@ -810,7 +824,7 @@ impl SourceManager { break worker; } Err(e) => { - tracing::warn!("failed to create source worker: {}", e); + tracing::warn!(error = %e.as_report(), "failed to create source worker"); } } }; @@ -839,7 +853,7 @@ impl SourceManager { source: &Source, managed_sources: &mut HashMap, metrics: Arc, - ) -> anyhow::Result<()> { + ) -> MetaResult<()> { tracing::info!("spawning new watcher for source {}", source.id); let splits = Arc::new(Mutex::new(SharedSplitMap { splits: None })); @@ -866,11 +880,12 @@ impl SourceManager { // in kafka tokio::time::timeout(Self::DEFAULT_SOURCE_TICK_TIMEOUT, worker.tick()) .await - .map_err(|_e| { - anyhow!( - "failed to fetch meta info for source {}, error: timeout {}", + .ok() + .with_context(|| { + format!( + "failed to fetch meta info for source {}, timeout {:?}", source.id, - Self::DEFAULT_SOURCE_TICK_TIMEOUT.as_secs() + Self::DEFAULT_SOURCE_TICK_TIMEOUT ) })??; @@ -926,8 +941,8 @@ impl SourceManager { let _pause_guard = self.paused.lock().await; if let Err(e) = self.tick().await { tracing::error!( - "error happened while running source manager tick: {}", - e.to_string() + error = %e.as_report(), + "error happened while running source manager tick", ); } } @@ -971,8 +986,8 @@ pub fn build_actor_split_impls( mod tests { use std::collections::{BTreeMap, HashMap, HashSet}; - use anyhow::anyhow; use risingwave_common::types::JsonbVal; + use risingwave_connector::error::ConnectorResult; use risingwave_connector::source::{SplitId, SplitMetaData}; use serde::{Deserialize, Serialize}; @@ -993,11 +1008,11 @@ mod tests { serde_json::to_value(*self).unwrap().into() } - fn restore_from_json(value: JsonbVal) -> anyhow::Result { - serde_json::from_value(value.take()).map_err(|e| anyhow!(e)) + fn restore_from_json(value: JsonbVal) -> ConnectorResult { + serde_json::from_value(value.take()).map_err(Into::into) } - fn update_with_offset(&mut self, _start_offset: String) -> anyhow::Result<()> { + fn update_with_offset(&mut self, _start_offset: String) -> ConnectorResult<()> { Ok(()) } } diff --git a/src/meta/src/stream/stream_graph/fragment.rs b/src/meta/src/stream/stream_graph/fragment.rs index 925b01c8cbdcf..fbe250818e6e4 100644 --- a/src/meta/src/stream/stream_graph/fragment.rs +++ b/src/meta/src/stream/stream_graph/fragment.rs @@ -51,10 +51,12 @@ pub(super) struct BuildingFragment { /// The fragment structure from the frontend, with the global fragment ID. inner: StreamFragment, - /// The ID of the job if it's materialized in this fragment. + /// The ID of the job if it contains the streaming job node. table_id: Option, /// The required columns of each upstream table. + /// + /// For shared CDC table on source, its `vec![]`, since the upstream source's output schema is fixed. upstream_table_columns: HashMap>, } @@ -145,6 +147,11 @@ impl BuildingFragment { has_table = true; } + NodeBody::Subscription(subscription_node) => { + subscription_node.subscription_catalog.as_mut().unwrap().id = table_id; + + has_table = true; + } NodeBody::Dml(dml_node) => { dml_node.table_id = table_id; dml_node.table_version_id = job.table_version_id().unwrap(); @@ -174,10 +181,7 @@ impl BuildingFragment { stream_scan.table_id.into(), stream_scan.upstream_column_ids.clone(), ), - NodeBody::CdcFilter(cdc_filter) => ( - cdc_filter.upstream_source_id.into(), - cdc_filter.upstream_column_ids.clone(), - ), + NodeBody::CdcFilter(cdc_filter) => (cdc_filter.upstream_source_id.into(), vec![]), _ => return, }; table_columns @@ -457,14 +461,14 @@ impl StreamFragmentGraph { Ok(()) } - /// Returns the fragment id where the table is materialized. + /// Returns the fragment id where the streaming job node located. pub fn table_fragment_id(&self) -> FragmentId { self.fragments .values() .filter(|b| b.table_id.is_some()) .map(|b| b.fragment_id) .exactly_one() - .expect("require exactly 1 materialize/sink node when creating the streaming job") + .expect("require exactly 1 materialize/sink/cdc source node when creating the streaming job") } /// Returns the fragment id where the table dml is received. @@ -655,7 +659,10 @@ impl CompleteStreamFragmentGraph { (source_job_id, edge) } - DdlType::MaterializedView | DdlType::Sink | DdlType::Index => { + DdlType::MaterializedView + | DdlType::Sink + | DdlType::Index + | DdlType::Subscription => { // handle MV on MV // Build the extra edges between the upstream `Materialize` and the downstream `StreamScan` @@ -670,22 +677,8 @@ impl CompleteStreamFragmentGraph { let nodes = mview_fragment.actors[0].get_nodes().unwrap(); let mview_node = nodes.get_node_body().unwrap().as_materialize().unwrap(); - let all_column_ids = mview_node - .get_table() - .unwrap() - .columns - .iter() - .map(|c| c.column_desc.as_ref().unwrap().column_id) - .collect_vec(); - let dist_key_indices: Vec<_> = mview_node - .table - .as_ref() - .unwrap() - .distribution_key - .iter() - .map(|i| *i as u32) - .collect(); - + let all_column_ids = mview_node.column_ids(); + let dist_key_indices = mview_node.dist_key_indices(); let output_indices = output_columns .iter() .map(|c| { @@ -700,27 +693,11 @@ impl CompleteStreamFragmentGraph { )?; (dist_key_indices, output_indices) }; - let dispatch_strategy = if uses_arrangement_backfill { - if !dist_key_indices.is_empty() { - DispatchStrategy { - r#type: DispatcherType::Hash as _, - dist_key_indices, - output_indices, - } - } else { - DispatchStrategy { - r#type: DispatcherType::Simple as _, - dist_key_indices: vec![], // empty for Simple - output_indices, - } - } - } else { - DispatchStrategy { - r#type: DispatcherType::NoShuffle as _, - dist_key_indices: vec![], // not used for `NoShuffle` - output_indices, - } - }; + let dispatch_strategy = mv_on_mv_dispatch_strategy( + uses_arrangement_backfill, + dist_key_indices, + output_indices, + ); let edge = StreamFragmentEdge { id: EdgeId::UpstreamExternal { upstream_table_id, @@ -806,6 +783,34 @@ impl CompleteStreamFragmentGraph { } } +fn mv_on_mv_dispatch_strategy( + uses_arrangement_backfill: bool, + dist_key_indices: Vec, + output_indices: Vec, +) -> DispatchStrategy { + if uses_arrangement_backfill { + if !dist_key_indices.is_empty() { + DispatchStrategy { + r#type: DispatcherType::Hash as _, + dist_key_indices, + output_indices, + } + } else { + DispatchStrategy { + r#type: DispatcherType::Simple as _, + dist_key_indices: vec![], // empty for Simple + output_indices, + } + } + } else { + DispatchStrategy { + r#type: DispatcherType::NoShuffle as _, + dist_key_indices: vec![], // not used for `NoShuffle` + output_indices, + } + } +} + impl CompleteStreamFragmentGraph { /// Returns **all** fragment IDs in the complete graph, including the ones that are not in the /// building graph. @@ -896,11 +901,17 @@ impl CompleteStreamFragmentGraph { } = building_fragment; let distribution_type = distribution.to_distribution_type() as i32; + let materialized_fragment_id = + if inner.fragment_type_mask & FragmentTypeFlag::Mview as u32 != 0 { + table_id + } else { + None + }; let state_table_ids = internal_tables .iter() .map(|t| t.id) - .chain(table_id) + .chain(materialized_fragment_id) .collect(); let upstream_fragment_ids = self diff --git a/src/meta/src/stream/stream_manager.rs b/src/meta/src/stream/stream_manager.rs index f10738d3f5cab..a949780e06ed1 100644 --- a/src/meta/src/stream/stream_manager.rs +++ b/src/meta/src/stream/stream_manager.rs @@ -12,36 +12,36 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::{BTreeMap, BTreeSet, HashMap, HashSet}; +use std::collections::{BTreeSet, HashMap, HashSet}; use std::sync::Arc; -use futures::future::{join_all, try_join_all, BoxFuture}; -use futures::stream::FuturesUnordered; -use futures::TryStreamExt; +use futures::future::{join_all, BoxFuture}; use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_pb::catalog::{CreateType, Table}; use risingwave_pb::stream_plan::update_mutation::MergeUpdate; use risingwave_pb::stream_plan::Dispatcher; -use risingwave_pb::stream_service::{ - BroadcastActorInfoTableRequest, BuildActorsRequest, DropActorsRequest, UpdateActorsRequest, -}; +use thiserror_ext::AsReport; use tokio::sync::mpsc::Sender; -use tokio::sync::{oneshot, Mutex, RwLock}; +use tokio::sync::{oneshot, Mutex}; use tracing::Instrument; -use uuid::Uuid; -use super::{Locations, RescheduleOptions, ScaleController, ScaleControllerRef, TableResizePolicy}; -use crate::barrier::{BarrierScheduler, Command, ReplaceTablePlan}; +use super::{Locations, RescheduleOptions, ScaleControllerRef, TableResizePolicy}; +use crate::barrier::{BarrierScheduler, Command, ReplaceTablePlan, StreamRpcManager}; use crate::hummock::HummockManagerRef; -use crate::manager::{DdlType, MetaSrvEnv, MetadataManager, StreamingJob, WorkerId}; -use crate::model::{ActorId, TableFragments, TableParallelism}; +use crate::manager::{DdlType, MetaSrvEnv, MetadataManager, StreamingJob}; +use crate::model::{ActorId, MetadataModel, TableFragments, TableParallelism}; use crate::stream::SourceManagerRef; use crate::{MetaError, MetaResult}; pub type GlobalStreamManagerRef = Arc; +#[derive(Default)] +pub struct CreateStreamingJobOption { + pub new_independent_compaction_group: bool, +} + /// [`CreateStreamingJobContext`] carries one-time infos for creating a streaming job. /// /// Note: for better readability, keep this struct complete and immutable once created. @@ -62,10 +62,6 @@ pub struct CreateStreamingJobContext { /// The locations of the existing actors, essentially the upstream mview actors to update. pub existing_locations: Locations, - /// The properties of the streaming job. - // TODO: directly store `StreamingJob` here. - pub table_properties: HashMap, - /// DDL definition. pub definition: String, @@ -77,6 +73,8 @@ pub struct CreateStreamingJobContext { /// Context provided for potential replace table, typically used when sinking into a table. pub replace_table_job_info: Option<(StreamingJob, ReplaceTableContext, TableFragments)>, + + pub option: CreateStreamingJobOption, } impl CreateStreamingJobContext { @@ -130,18 +128,18 @@ impl CreatingStreamingJobInfo { let mut receivers = HashMap::new(); let mut recovered_job_ids = vec![]; for job_id in job_ids { - if let Some(job) = jobs.get_mut(&job_id) - && let Some(shutdown_tx) = job.shutdown_tx.take() - { - let (tx, rx) = oneshot::channel(); - if shutdown_tx - .send(CreatingState::Canceling { finish_tx: tx }) - .await - .is_ok() - { - receivers.insert(job_id, rx); - } else { - tracing::warn!(id=?job_id, "failed to send canceling state"); + if let Some(job) = jobs.get_mut(&job_id) { + if let Some(shutdown_tx) = job.shutdown_tx.take() { + let (tx, rx) = oneshot::channel(); + if shutdown_tx + .send(CreatingState::Canceling { finish_tx: tx }) + .await + .is_ok() + { + receivers.insert(job_id, rx); + } else { + tracing::warn!(id=?job_id, "failed to send canceling state"); + } } } else { // If these job ids do not exist in streaming_jobs, @@ -175,10 +173,6 @@ pub struct ReplaceTableContext { /// The locations of the existing actors, essentially the downstream chain actors to update. pub existing_locations: Locations, - - /// The properties of the streaming job. - // TODO: directly store `StreamingJob here. - pub table_properties: HashMap, } /// `GlobalStreamManager` manages all the streams in the system. @@ -198,9 +192,9 @@ pub struct GlobalStreamManager { hummock_manager: HummockManagerRef, - pub reschedule_lock: RwLock<()>, + pub scale_controller: ScaleControllerRef, - pub(crate) scale_controller: Option, + pub stream_rpc_manager: StreamRpcManager, } impl GlobalStreamManager { @@ -210,15 +204,9 @@ impl GlobalStreamManager { barrier_scheduler: BarrierScheduler, source_manager: SourceManagerRef, hummock_manager: HummockManagerRef, + stream_rpc_manager: StreamRpcManager, + scale_controller: ScaleControllerRef, ) -> MetaResult { - let scale_controller = match &metadata_manager { - MetadataManager::V1(_) => { - let scale_controller = - ScaleController::new(&metadata_manager, source_manager.clone(), env.clone()); - Some(Arc::new(scale_controller)) - } - MetadataManager::V2(_) => None, - }; Ok(Self { env, metadata_manager, @@ -226,8 +214,8 @@ impl GlobalStreamManager { source_manager, hummock_manager, creating_job_info: Arc::new(CreatingStreamingJobInfo::default()), - reschedule_lock: RwLock::new(()), scale_controller, + stream_rpc_manager, }) } @@ -272,7 +260,7 @@ impl GlobalStreamManager { }) .await .inspect_err(|_| { - tracing::warn!("failed to notify failed: {table_id}, err: {err}") + tracing::warn!(error = %err.as_report(), "failed to notify failed: {table_id}") }); } } @@ -296,49 +284,32 @@ impl GlobalStreamManager { .await { // try to cancel buffered creating command. - if self - .barrier_scheduler - .try_cancel_scheduled_create(table_id) - .await - { + if self.barrier_scheduler.try_cancel_scheduled_create(table_id) { tracing::debug!( "cancelling streaming job {table_id} in buffer queue." ); let node_actors = table_fragments.worker_actor_ids(); let cluster_info = self.metadata_manager.get_streaming_cluster_info().await?; - let node_actors = node_actors - .into_iter() - .map(|(id, actor_ids)| { - ( - cluster_info.worker_nodes.get(&id).cloned().unwrap(), - actor_ids, - ) - }) - .collect_vec(); - let futures = node_actors.into_iter().map(|(node, actor_ids)| { - let request_id = Uuid::new_v4().to_string(); - async move { - let client = - self.env.stream_client_pool().get(&node).await?; - let request = DropActorsRequest { - request_id, - actor_ids, - }; - client.drop_actors(request).await - } - }); - try_join_all(futures).await?; - - self.metadata_manager - .drop_streaming_job_by_ids(&HashSet::from_iter( - std::iter::once(table_id), - )) + self.stream_rpc_manager + .drop_actors( + &cluster_info.worker_nodes, + node_actors.into_iter(), + ) .await?; + + if let MetadataManager::V1(mgr) = &self.metadata_manager { + mgr.fragment_manager + .drop_table_fragments_vec(&HashSet::from_iter( + std::iter::once(table_id), + )) + .await?; + } } else if !table_fragments.is_created() { tracing::debug!( "cancelling streaming job {table_id} by issue cancel command." ); + self.barrier_scheduler .run_command(Command::CancelStreamingJob(table_fragments)) .await?; @@ -365,7 +336,6 @@ impl GlobalStreamManager { res } - /// First broadcasts the actor info to `WorkerNodes`, and then let them build actors and channels. async fn build_actors( &self, table_fragments: &TableFragments, @@ -380,8 +350,7 @@ impl GlobalStreamManager { // 2. all upstream actors. let actor_infos_to_broadcast = building_locations .actor_infos() - .chain(existing_locations.actor_infos()) - .collect_vec(); + .chain(existing_locations.actor_infos()); let building_worker_actors = building_locations.worker_actors(); @@ -389,57 +358,28 @@ impl GlobalStreamManager { // The first stage does 2 things: broadcast actor info, and send local actor ids to // different WorkerNodes. Such that each WorkerNode knows the overall actor // allocation, but not actually builds it. We initialize all channels in this stage. - building_worker_actors.iter().map(|(worker_id, actors)| { - let stream_actors = actors - .iter() - .map(|actor_id| actor_map[actor_id].clone()) - .collect::>(); - let worker_node = building_locations.worker_locations.get(worker_id).unwrap(); - let actor_infos_to_broadcast = &actor_infos_to_broadcast; - async move { - let client = self.env.stream_client_pool().get(worker_node).await?; - - client - .broadcast_actor_info_table(BroadcastActorInfoTableRequest { - info: actor_infos_to_broadcast.clone(), - }) - .await?; - - let request_id = Uuid::new_v4().to_string(); - tracing::debug!(request_id = request_id.as_str(), actors = ?actors, "update actors"); - client - .update_actors(UpdateActorsRequest { - request_id, - actors: stream_actors.clone(), - }) - .await?; - - Ok(()) as MetaResult<_> - } - }).collect::>().try_collect::<()>().await?; + self.stream_rpc_manager + .broadcast_update_actor_info( + &building_locations.worker_locations, + building_worker_actors.keys().cloned(), + actor_infos_to_broadcast, + building_worker_actors.iter().map(|(worker_id, actors)| { + let stream_actors = actors + .iter() + .map(|actor_id| actor_map[actor_id].clone()) + .collect::>(); + (*worker_id, stream_actors) + }), + ) + .await?; // In the second stage, each [`WorkerNode`] builds local actors and connect them with // channels. - building_worker_actors - .iter() - .map(|(worker_id, actors)| async move { - let worker_node = building_locations.worker_locations.get(worker_id).unwrap(); - - let client = self.env.stream_client_pool().get(worker_node).await?; - - let request_id = Uuid::new_v4().to_string(); - tracing::debug!(request_id = request_id.as_str(), actors = ?actors, "build actors"); - client - .build_actors(BuildActorsRequest { - request_id, - actor_id: actors.clone(), - }) - .await?; - - Ok(()) as MetaResult<()> - }) - .collect::>() - .try_collect::<()>() + self.stream_rpc_manager + .build_actors( + &building_locations.worker_locations, + building_worker_actors.into_iter(), + ) .await?; Ok(()) @@ -452,7 +392,6 @@ impl GlobalStreamManager { CreateStreamingJobContext { dispatchers, upstream_mview_actors, - table_properties, building_locations, existing_locations, definition, @@ -461,6 +400,7 @@ impl GlobalStreamManager { create_type, ddl_type, replace_table_job_info, + option, }: CreateStreamingJobContext, ) -> MetaResult<()> { let mut replace_table_command = None; @@ -472,7 +412,7 @@ impl GlobalStreamManager { .register_table_fragments( mv_table_id, internal_tables.keys().copied().collect(), - &table_properties, + option, ) .await?; debug_assert_eq!( @@ -490,10 +430,7 @@ impl GlobalStreamManager { self.build_actors(&table_fragments, &building_locations, &existing_locations) .await?; - if let Some((_, context, table_fragments)) = replace_table_job_info { - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support create sink into table in v2"); - }; + if let Some((streaming_job, context, table_fragments)) = replace_table_job_info { self.build_actors( &table_fragments, &context.building_locations, @@ -501,10 +438,19 @@ impl GlobalStreamManager { ) .await?; - // Add table fragments to meta store with state: `State::Initial`. - mgr.fragment_manager - .start_create_table_fragments(table_fragments.clone()) - .await?; + match &self.metadata_manager { + MetadataManager::V1(mgr) => { + // Add table fragments to meta store with state: `State::Initial`. + mgr.fragment_manager + .start_create_table_fragments(table_fragments.clone()) + .await? + } + MetadataManager::V2(mgr) => { + mgr.catalog_controller + .prepare_streaming_job(table_fragments.to_protobuf(), &streaming_job, true) + .await? + } + } let dummy_table_id = table_fragments.table_id(); @@ -537,14 +483,16 @@ impl GlobalStreamManager { }; if let Err(err) = self.barrier_scheduler.run_command(command).await { - if create_type == CreateType::Foreground { + if create_type == CreateType::Foreground || err.is_cancelled() { let mut table_ids = HashSet::from_iter(std::iter::once(table_id)); if let Some(dummy_table_id) = replace_table_id { table_ids.insert(dummy_table_id); } - self.metadata_manager - .drop_streaming_job_by_ids(&table_ids) - .await?; + if let MetadataManager::V1(mgr) = &self.metadata_manager { + mgr.fragment_manager + .drop_table_fragments_vec(&table_ids) + .await?; + } } return Err(err); @@ -562,7 +510,6 @@ impl GlobalStreamManager { dispatchers, building_locations, existing_locations, - table_properties: _, }: ReplaceTableContext, ) -> MetaResult<()> { self.build_actors(&table_fragments, &building_locations, &existing_locations) @@ -582,9 +529,10 @@ impl GlobalStreamManager { init_split_assignment, })) .await + && let MetadataManager::V1(mgr) = &self.metadata_manager { - self.metadata_manager - .drop_streaming_job_by_ids(&HashSet::from_iter(std::iter::once(dummy_table_id))) + mgr.fragment_manager + .drop_table_fragments_vec(&HashSet::from_iter(std::iter::once(dummy_table_id))) .await?; return Err(err); } @@ -601,15 +549,33 @@ impl GlobalStreamManager { .drop_streaming_jobs_impl(streaming_job_ids) .await .inspect_err(|err| { - tracing::error!(error = ?err, "Failed to drop streaming jobs"); + tracing::error!(error = ?err.as_report(), "Failed to drop streaming jobs"); }); } } + pub async fn drop_streaming_jobs_v2( + &self, + removed_actors: Vec, + state_table_ids: Vec, + ) { + if !removed_actors.is_empty() { + let _ = self + .barrier_scheduler + .run_command(Command::DropStreamingJobs(removed_actors)) + .await + .inspect_err(|err| { + tracing::error!(error = ?err.as_report(), "failed to run drop command"); + }); + + self.hummock_manager + .unregister_table_ids_fail_fast(&state_table_ids) + .await; + } + } + pub async fn drop_streaming_jobs_impl(&self, table_ids: Vec) -> MetaResult<()> { - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("call drop_streaming_jobs_impl_v2 instead.") - }; + let mgr = self.metadata_manager.as_v1_ref(); let table_fragments_vec = mgr .fragment_manager .select_table_fragments_by_ids(&table_ids) @@ -619,10 +585,24 @@ impl GlobalStreamManager { .drop_source_fragments(&table_fragments_vec) .await; - self.barrier_scheduler - .run_command(Command::DropStreamingJobs(table_ids.into_iter().collect())) + // Drop table fragments directly. + mgr.fragment_manager + .drop_table_fragments_vec(&table_ids.into_iter().collect()) .await?; + // Issues a drop barrier command. + let dropped_actors = table_fragments_vec + .iter() + .flat_map(|tf| tf.actor_ids().into_iter()) + .collect_vec(); + let _ = self + .barrier_scheduler + .run_command(Command::DropStreamingJobs(dropped_actors)) + .await + .inspect_err(|err| { + tracing::error!(error = ?err.as_report(), "failed to run drop command"); + }); + // Unregister from compaction group afterwards. self.hummock_manager .unregister_table_fragments_vec(&table_fragments_vec) @@ -631,24 +611,6 @@ impl GlobalStreamManager { Ok(()) } - pub fn drop_streaming_jobs_impl_v2( - &self, - _job_info: HashMap>>, - _state_table_ids: Vec, - ) -> MetaResult<()> { - // self.barrier_scheduler.run_command(Command::DropStreamingJobsV2(job_info)).await?; - // - // // TODO: need some refactoring on source manager. - // - // // Unregister from compaction group afterwards. - // self.hummock_manager - // .unregister_table_ids_fail_fast( - // &state_table_ids - // ) - // .await; - unimplemented!("drop_streaming_jobs_impl_v2") - } - /// Cancel streaming jobs and return the canceled table ids. /// 1. Send cancel message to stream jobs (via `cancel_jobs`). /// 2. Send cancel message to recovered stream jobs (via `barrier_scheduler`). @@ -660,7 +622,7 @@ impl GlobalStreamManager { return vec![]; } - let _reschedule_job_lock = self.reschedule_lock.read().await; + let _reschedule_job_lock = self.reschedule_lock_read_guard().await; let (receivers, recovered_job_ids) = self.creating_job_info.cancel_jobs(table_ids).await; let futures = receivers.into_iter().map(|(id, receiver)| async move { @@ -688,10 +650,9 @@ impl GlobalStreamManager { id )))?; } - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support cancel streaming job in v2"); - }; - mgr.catalog_manager.cancel_create_table_procedure(id.into(), fragment.internal_table_ids()).await?; + if let MetadataManager::V1(mgr) = &self.metadata_manager { + mgr.catalog_manager.cancel_create_table_procedure(id.into(), fragment.internal_table_ids()).await?; + } self.barrier_scheduler .run_command(Command::CancelStreamingJob(fragment)) @@ -702,8 +663,8 @@ impl GlobalStreamManager { tracing::info!(?id, "cancelled recovered streaming job"); Some(id) } - Err(_) => { - tracing::error!(?id, "failed to cancel recovered streaming job, does it correspond to any jobs in `SHOW JOBS`?"); + Err(err) => { + tracing::error!(error=?err.as_report(), "failed to cancel recovered streaming job {id}, does it correspond to any jobs in `SHOW JOBS`?"); None } } @@ -718,16 +679,14 @@ impl GlobalStreamManager { &self, table_id: u32, parallelism: TableParallelism, + deferred: bool, ) -> MetaResult<()> { - let MetadataManager::V1(mgr) = &self.metadata_manager else { - unimplemented!("support alter table parallelism in v2"); - }; - let _reschedule_job_lock = self.reschedule_lock.write().await; + let _reschedule_job_lock = self.reschedule_lock_write_guard().await; - let worker_nodes = mgr - .cluster_manager + let worker_nodes = self + .metadata_manager .list_active_streaming_compute_nodes() - .await; + .await?; let worker_ids = worker_nodes .iter() @@ -735,24 +694,45 @@ impl GlobalStreamManager { .map(|node| node.id) .collect::>(); - let reschedules = self - .scale_controller - .as_ref() - .unwrap() - .generate_table_resize_plan(TableResizePolicy { - worker_ids, - table_parallelisms: vec![(table_id, parallelism)].into_iter().collect(), - }) - .await?; + let table_parallelism_assignment = HashMap::from([(TableId::new(table_id), parallelism)]); - self.reschedule_actors( - reschedules, - RescheduleOptions { - resolve_no_shuffle_upstream: false, - }, - Some(HashMap::from([(TableId::new(table_id), parallelism)])), - ) - .await?; + if deferred { + tracing::debug!( + "deferred mode enabled for job {}, set the parallelism directly to {:?}", + table_id, + parallelism + ); + self.scale_controller + .post_apply_reschedule(&HashMap::new(), &table_parallelism_assignment) + .await?; + } else { + let reschedules = self + .scale_controller + .generate_table_resize_plan(TableResizePolicy { + worker_ids, + table_parallelisms: table_parallelism_assignment + .iter() + .map(|(id, parallelism)| (id.table_id, *parallelism)) + .collect(), + }) + .await?; + + if reschedules.is_empty() { + tracing::debug!("empty reschedule plan generated for job {}, set the parallelism directly to {:?}", table_id, parallelism); + self.scale_controller + .post_apply_reschedule(&HashMap::new(), &table_parallelism_assignment) + .await?; + } else { + self.reschedule_actors( + reschedules, + RescheduleOptions { + resolve_no_shuffle_upstream: false, + }, + Some(table_parallelism_assignment), + ) + .await?; + } + }; Ok(()) } @@ -767,6 +747,7 @@ mod tests { use risingwave_common::catalog::TableId; use risingwave_common::hash::ParallelUnitMapping; + use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_pb::common::{HostAddress, WorkerType}; use risingwave_pb::meta::add_worker_node_request::Property; use risingwave_pb::meta::table_fragments::fragment::FragmentDistributionType; @@ -788,7 +769,7 @@ mod tests { use tonic::{Request, Response, Status}; use super::*; - use crate::barrier::GlobalBarrierManager; + use crate::barrier::{GlobalBarrierManager, StreamRpcManager}; use crate::hummock::{CompactorManager, HummockManager}; use crate::manager::sink_coordination::SinkCoordinatorManager; use crate::manager::{ @@ -798,7 +779,7 @@ mod tests { use crate::model::{ActorId, FragmentId}; use crate::rpc::ddl_controller::DropMode; use crate::rpc::metrics::MetaMetrics; - use crate::stream::SourceManager; + use crate::stream::{ScaleController, SourceManager}; use crate::MetaOpts; struct FakeFragmentState { @@ -815,7 +796,7 @@ mod tests { impl StreamService for FakeStreamService { async fn update_actors( &self, - request: Request, + request: Request, ) -> std::result::Result, Status> { let req = request.into_inner(); let mut guard = self.inner.actor_streams.lock().unwrap(); @@ -844,7 +825,7 @@ mod tests { async fn broadcast_actor_info_table( &self, - request: Request, + request: Request, ) -> std::result::Result, Status> { let req = request.into_inner(); let mut guard = self.inner.actor_infos.lock().unwrap(); @@ -996,6 +977,14 @@ mod tests { let (sink_manager, _) = SinkCoordinatorManager::start_worker(); + let stream_rpc_manager = StreamRpcManager::new(env.clone()); + let scale_controller = Arc::new(ScaleController::new( + &metadata_manager, + source_manager.clone(), + stream_rpc_manager.clone(), + env.clone(), + )); + let barrier_manager = GlobalBarrierManager::new( scheduled_barriers, env.clone(), @@ -1004,6 +993,8 @@ mod tests { source_manager.clone(), sink_manager, meta_metrics.clone(), + stream_rpc_manager.clone(), + scale_controller.clone(), ); let stream_manager = GlobalStreamManager::new( @@ -1012,6 +1003,8 @@ mod tests { barrier_scheduler.clone(), source_manager.clone(), hummock_manager, + stream_rpc_manager, + scale_controller.clone(), )?; let (join_handle_2, shutdown_tx_2) = GlobalBarrierManager::start(barrier_manager); @@ -1056,7 +1049,9 @@ mod tests { let actor_locations = fragments .values() .flat_map(|f| &f.actors) - .map(|a| (a.actor_id, parallel_units[&0].clone())) + .sorted_by(|a, b| a.actor_id.cmp(&b.actor_id)) + .enumerate() + .map(|(idx, a)| (a.actor_id, parallel_units[&(idx as u32)].clone())) .collect(); Locations { @@ -1074,7 +1069,7 @@ mod tests { fragments, &locations.actor_locations, Default::default(), - TableParallelism::Auto, + TableParallelism::Adaptive, ); let ctx = CreateStreamingJobContext { building_locations: locations, @@ -1145,6 +1140,14 @@ mod tests { let table_id = TableId::new(0); let actors = make_mview_stream_actors(&table_id, 4); + let StreamingClusterInfo { parallel_units, .. } = services + .global_stream_manager + .metadata_manager + .get_streaming_cluster_info() + .await?; + + let parallel_unit_ids = parallel_units.keys().cloned().sorted().collect_vec(); + let mut fragments = BTreeMap::default(); fragments.insert( 0, @@ -1154,7 +1157,9 @@ mod tests { distribution_type: FragmentDistributionType::Hash as i32, actors: actors.clone(), state_table_ids: vec![0], - vnode_mapping: Some(ParallelUnitMapping::new_single(0).to_protobuf()), + vnode_mapping: Some( + ParallelUnitMapping::new_uniform(parallel_unit_ids.into_iter()).to_protobuf(), + ), ..Default::default() }, ); diff --git a/src/meta/src/stream/test_scale.rs b/src/meta/src/stream/test_scale.rs index 2db55dbddbd4d..73d59ff52f2f4 100644 --- a/src/meta/src/stream/test_scale.rs +++ b/src/meta/src/stream/test_scale.rs @@ -21,10 +21,10 @@ mod tests { use risingwave_common::buffer::Bitmap; use risingwave_common::hash::{ActorMapping, ParallelUnitId, ParallelUnitMapping, VirtualNode}; use risingwave_pb::common::ParallelUnit; - use risingwave_pb::stream_plan::StreamActor; use crate::model::ActorId; use crate::stream::scale::rebalance_actor_vnode; + use crate::stream::CustomActorInfo; fn simulated_parallel_unit_nums(min: Option, max: Option) -> Vec { let mut raw = vec![1, 3, 12, 42, VirtualNode::COUNT]; @@ -39,13 +39,13 @@ mod tests { raw } - fn build_fake_actors(info: &[(ActorId, ParallelUnitId)]) -> Vec { + fn build_fake_actors(info: &[(ActorId, ParallelUnitId)]) -> Vec { let parallel_units = generate_parallel_units(info); let vnode_bitmaps = ParallelUnitMapping::build(¶llel_units).to_bitmaps(); info.iter() - .map(|(actor_id, parallel_unit_id)| StreamActor { + .map(|(actor_id, parallel_unit_id)| CustomActorInfo { actor_id: *actor_id, vnode_bitmap: vnode_bitmaps .get(parallel_unit_id) @@ -64,7 +64,7 @@ mod tests { .collect_vec() } - fn check_affinity_for_scale_in(bitmap: &Bitmap, actor: &StreamActor) { + fn check_affinity_for_scale_in(bitmap: &Bitmap, actor: &CustomActorInfo) { let prev_bitmap = Bitmap::from(actor.vnode_bitmap.as_ref().unwrap()); for idx in 0..VirtualNode::COUNT { diff --git a/src/meta/src/telemetry.rs b/src/meta/src/telemetry.rs index fdb0ed06fd769..43533f96ae10b 100644 --- a/src/meta/src/telemetry.rs +++ b/src/meta/src/telemetry.rs @@ -18,6 +18,7 @@ use risingwave_common::telemetry::{ current_timestamp, SystemData, TelemetryNodeType, TelemetryReport, TelemetryReportBase, TelemetryResult, }; +use risingwave_common::{GIT_SHA, RW_VERSION}; use risingwave_pb::common::WorkerType; use serde::{Deserialize, Serialize}; use thiserror_ext::AsReport; @@ -33,13 +34,21 @@ struct NodeCount { compactor_count: u64, } +#[derive(Debug, Serialize, Deserialize)] +struct RwVersion { + version: String, + git_sha: String, +} + #[derive(Debug, Serialize, Deserialize)] pub struct MetaTelemetryReport { #[serde(flatten)] base: TelemetryReportBase, node_count: NodeCount, + streaming_job_count: u64, // At this point, it will always be etcd, but we will enable telemetry when using memory. meta_backend: MetaBackend, + rw_version: RwVersion, } impl TelemetryReport for MetaTelemetryReport {} @@ -91,7 +100,17 @@ impl TelemetryReportCreator for MetaReportCreator { .await .map_err(|err| err.as_report().to_string())?; + let streaming_job_count = self + .metadata_manager + .count_streaming_job() + .await + .map_err(|err| err.as_report().to_string())? as u64; + Ok(MetaTelemetryReport { + rw_version: RwVersion { + version: RW_VERSION.to_string(), + git_sha: GIT_SHA.to_string(), + }, base: TelemetryReportBase { tracking_id, session_id, @@ -106,6 +125,7 @@ impl TelemetryReportCreator for MetaReportCreator { frontend_count: *node_map.get(&WorkerType::Frontend).unwrap_or(&0), compactor_count: *node_map.get(&WorkerType::Compactor).unwrap_or(&0), }, + streaming_job_count, meta_backend: self.meta_backend, }) } diff --git a/src/object_store/Cargo.toml b/src/object_store/Cargo.toml index 95354557dd943..5acc52937f4ba 100644 --- a/src/object_store/Cargo.toml +++ b/src/object_store/Cargo.toml @@ -18,7 +18,7 @@ aws-smithy-runtime = { workspace = true } aws-smithy-runtime-api = { workspace = true } aws-smithy-types = { workspace = true } bytes = { version = "1", features = ["serde"] } -crc32fast = "1.3.2" +crc32fast = "1" either = "1" fail = "0.5" futures = { version = "0.3", default-features = false, features = ["alloc"] } @@ -26,7 +26,8 @@ hyper = { version = "0.14", features = ["tcp", "client"] } hyper-rustls = { version = "0.24.2", features = ["webpki-roots"] } hyper-tls = "0.5.0" itertools = "0.12" -opendal = "0.44" +madsim = "0.2.22" +opendal = "0.44.2" prometheus = { version = "0.13", features = ["process"] } risingwave_common = { workspace = true } rustls = "0.21.8" diff --git a/src/object_store/src/object/error.rs b/src/object_store/src/object/error.rs index c84527e6bdb03..f13ae2b4d8f51 100644 --- a/src/object_store/src/object/error.rs +++ b/src/object_store/src/object/error.rs @@ -15,19 +15,19 @@ use std::io; use std::marker::{Send, Sync}; -use aws_sdk_s3::error::DisplayErrorContext; use aws_sdk_s3::operation::get_object::GetObjectError; use aws_sdk_s3::operation::head_object::HeadObjectError; use aws_sdk_s3::primitives::ByteStreamError; use aws_smithy_types::body::SdkBody; use risingwave_common::error::BoxedError; use thiserror::Error; +use thiserror_ext::AsReport; use tokio::sync::oneshot::error::RecvError; #[derive(Error, Debug, thiserror_ext::Box, thiserror_ext::Construct)] #[thiserror_ext(newtype(name = ObjectError, backtrace, report_debug))] pub enum ObjectErrorInner { - #[error("s3 error: {}", DisplayErrorContext(&**.0))] + #[error("s3 error: {0}")] S3(#[source] BoxedError), #[error("disk error: {msg}")] Disk { @@ -42,6 +42,9 @@ pub enum ObjectErrorInner { #[error("Internal error: {0}")] #[construct(skip)] Internal(String), + #[cfg(madsim)] + #[error(transparent)] + Sim(#[from] crate::object::sim::SimError), } impl ObjectError { @@ -79,6 +82,10 @@ impl ObjectError { ObjectErrorInner::Mem(e) => { return e.is_object_not_found_error(); } + #[cfg(madsim)] + ObjectErrorInner::Sim(e) => { + return e.is_object_not_found_error(); + } _ => {} }; false @@ -97,12 +104,19 @@ where impl From for ObjectError { fn from(e: RecvError) -> Self { - ObjectErrorInner::Internal(e.to_string()).into() + ObjectErrorInner::Internal(e.to_report_string()).into() } } impl From for ObjectError { fn from(e: ByteStreamError) -> Self { + ObjectErrorInner::Internal(e.to_report_string()).into() + } +} + +#[cfg(madsim)] +impl From for ObjectError { + fn from(e: std::io::Error) -> Self { ObjectErrorInner::Internal(e.to_string()).into() } } diff --git a/src/object_store/src/object/mod.rs b/src/object_store/src/object/mod.rs index 32adba09d2bb1..df330967d9679 100644 --- a/src/object_store/src/object/mod.rs +++ b/src/object_store/src/object/mod.rs @@ -12,6 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#[cfg(madsim)] +pub mod sim; use std::ops::{Range, RangeBounds}; use std::sync::Arc; use std::time::Duration; @@ -37,6 +39,10 @@ pub mod object_metrics; pub use error::*; use object_metrics::ObjectStoreMetrics; +use thiserror_ext::AsReport; + +#[cfg(madsim)] +use self::sim::SimObjectStore; pub type ObjectStoreRef = Arc; pub type ObjectStreamingUploader = MonitoredStreamingUploader; @@ -112,32 +118,28 @@ pub trait ObjectStore: Send + Sync { /// specified in the request is not found, it will be considered as successfully deleted. async fn delete_objects(&self, paths: &[String]) -> ObjectResult<()>; - fn monitored(self, metrics: Arc) -> MonitoredObjectStore + fn monitored( + self, + metrics: Arc, + config: ObjectStoreConfig, + ) -> MonitoredObjectStore where Self: Sized, { - MonitoredObjectStore::new(self, metrics) + MonitoredObjectStore::new(self, metrics, config) } async fn list(&self, prefix: &str) -> ObjectResult; fn store_media_type(&self) -> &'static str; - - fn recv_buffer_size(&self) -> usize { - // 2MB - 1 << 21 - } - - fn config(&self) -> Option<&ObjectStoreConfig> { - // TODO: remove option - None - } } pub enum ObjectStoreImpl { InMem(MonitoredObjectStore), Opendal(MonitoredObjectStore), S3(MonitoredObjectStore), + #[cfg(madsim)] + Sim(MonitoredObjectStore), } macro_rules! dispatch_async { @@ -164,7 +166,12 @@ macro_rules! object_store_impl_method_body { ObjectStoreImpl::S3(s3) => { $dispatch_macro!(s3, $method_name, path $(, $args)*) }, + #[cfg(madsim)] + ObjectStoreImpl::Sim(in_mem) => { + $dispatch_macro!(in_mem, $method_name, path $(, $args)*) + }, } + } }; } @@ -178,6 +185,7 @@ macro_rules! object_store_impl_method_body_slice { ($object_store:expr, $method_name:ident, $dispatch_macro:ident, $paths:expr $(, $args:expr)*) => { { let paths_rem = partition_object_store_paths($paths); + match $object_store { ObjectStoreImpl::InMem(in_mem) => { $dispatch_macro!(in_mem, $method_name, &paths_rem $(, $args)*) @@ -188,6 +196,10 @@ macro_rules! object_store_impl_method_body_slice { ObjectStoreImpl::S3(s3) => { $dispatch_macro!(s3, $method_name, &paths_rem $(, $args)*) }, + #[cfg(madsim)] + ObjectStoreImpl::Sim(in_mem) => { + $dispatch_macro!(in_mem, $method_name, &paths_rem $(, $args)*) + }, } } }; @@ -246,6 +258,8 @@ impl ObjectStoreImpl { ObjectStoreImpl::InMem(store) => store.inner.get_object_prefix(obj_id), ObjectStoreImpl::Opendal(store) => store.inner.get_object_prefix(obj_id), ObjectStoreImpl::S3(store) => store.inner.get_object_prefix(obj_id), + #[cfg(madsim)] + ObjectStoreImpl::Sim(store) => store.inner.get_object_prefix(obj_id), } } @@ -256,14 +270,8 @@ impl ObjectStoreImpl { store.inner.op.info().native_capability().write_can_multi } ObjectStoreImpl::S3(_) => true, - } - } - - pub fn recv_buffer_size(&self) -> usize { - match self { - ObjectStoreImpl::InMem(store) => store.recv_buffer_size(), - ObjectStoreImpl::Opendal(store) => store.recv_buffer_size(), - ObjectStoreImpl::S3(store) => store.recv_buffer_size(), + #[cfg(madsim)] + ObjectStoreImpl::Sim(_) => true, } } } @@ -274,7 +282,7 @@ fn try_update_failure_metric( operation_type: &'static str, ) { if let Err(e) = &result { - tracing::error!("{:?} failed because of: {:?}", operation_type, e); + tracing::error!(error = %e.as_report(), "{} failed", operation_type); metrics .failure_count .with_label_values(&[operation_type]) @@ -502,29 +510,22 @@ pub struct MonitoredObjectStore { /// - start `operation_latency` timer /// - `failure-count` impl MonitoredObjectStore { - pub fn new(store: OS, object_store_metrics: Arc) -> Self { - if let Some(config) = store.config() { - Self { - object_store_metrics, - streaming_read_timeout: Some(Duration::from_millis( - config.object_store_streaming_read_timeout_ms, - )), - streaming_upload_timeout: Some(Duration::from_millis( - config.object_store_streaming_upload_timeout_ms, - )), - read_timeout: Some(Duration::from_millis(config.object_store_read_timeout_ms)), - upload_timeout: Some(Duration::from_millis(config.object_store_upload_timeout_ms)), - inner: store, - } - } else { - Self { - inner: store, - object_store_metrics, - streaming_read_timeout: None, - streaming_upload_timeout: None, - read_timeout: None, - upload_timeout: None, - } + pub fn new( + store: OS, + object_store_metrics: Arc, + config: ObjectStoreConfig, + ) -> Self { + Self { + object_store_metrics, + streaming_read_timeout: Some(Duration::from_millis( + config.object_store_streaming_read_timeout_ms, + )), + streaming_upload_timeout: Some(Duration::from_millis( + config.object_store_streaming_upload_timeout_ms, + )), + read_timeout: Some(Duration::from_millis(config.object_store_read_timeout_ms)), + upload_timeout: Some(Duration::from_millis(config.object_store_upload_timeout_ms)), + inner: store, } } @@ -776,10 +777,6 @@ impl MonitoredObjectStore { try_update_failure_metric(&self.object_store_metrics, &res, operation_type); res } - - fn recv_buffer_size(&self) -> usize { - self.inner.recv_buffer_size() - } } /// Creates a new [`ObjectStore`] from the given `url`. Credentials are configured via environment @@ -794,15 +791,27 @@ pub async fn build_remote_object_store( config: ObjectStoreConfig, ) -> ObjectStoreImpl { match url { - s3 if s3.starts_with("s3://") => ObjectStoreImpl::S3( - S3ObjectStore::new_with_config( - s3.strip_prefix("s3://").unwrap().to_string(), - metrics.clone(), - config, - ) - .await - .monitored(metrics), - ), + s3 if s3.starts_with("s3://") => { + if std::env::var("RW_USE_OPENDAL_FOR_S3").is_ok() { + let bucket = s3.strip_prefix("s3://").unwrap(); + + ObjectStoreImpl::Opendal( + OpendalObjectStore::new_s3_engine(bucket.to_string(), config.clone()) + .unwrap() + .monitored(metrics, config), + ) + } else { + ObjectStoreImpl::S3( + S3ObjectStore::new_with_config( + s3.strip_prefix("s3://").unwrap().to_string(), + metrics.clone(), + config.clone(), + ) + .await + .monitored(metrics, config), + ) + } + } #[cfg(feature = "hdfs-backend")] hdfs if hdfs.starts_with("hdfs://") => { let hdfs = hdfs.strip_prefix("hdfs://").unwrap(); @@ -819,7 +828,7 @@ pub async fn build_remote_object_store( ObjectStoreImpl::Opendal( OpendalObjectStore::new_gcs_engine(bucket.to_string(), root.to_string()) .unwrap() - .monitored(metrics), + .monitored(metrics, config), ) } obs if obs.starts_with("obs://") => { @@ -828,7 +837,7 @@ pub async fn build_remote_object_store( ObjectStoreImpl::Opendal( OpendalObjectStore::new_obs_engine(bucket.to_string(), root.to_string()) .unwrap() - .monitored(metrics), + .monitored(metrics, config), ) } @@ -838,7 +847,7 @@ pub async fn build_remote_object_store( ObjectStoreImpl::Opendal( OpendalObjectStore::new_oss_engine(bucket.to_string(), root.to_string()) .unwrap() - .monitored(metrics), + .monitored(metrics, config), ) } webhdfs if webhdfs.starts_with("webhdfs://") => { @@ -847,7 +856,7 @@ pub async fn build_remote_object_store( ObjectStoreImpl::Opendal( OpendalObjectStore::new_webhdfs_engine(namenode.to_string(), root.to_string()) .unwrap() - .monitored(metrics), + .monitored(metrics, config), ) } azblob if azblob.starts_with("azblob://") => { @@ -856,7 +865,7 @@ pub async fn build_remote_object_store( ObjectStoreImpl::Opendal( OpendalObjectStore::new_azblob_engine(container_name.to_string(), root.to_string()) .unwrap() - .monitored(metrics), + .monitored(metrics, config), ) } fs if fs.starts_with("fs://") => { @@ -864,7 +873,7 @@ pub async fn build_remote_object_store( ObjectStoreImpl::Opendal( OpendalObjectStore::new_fs_engine(fs.to_string()) .unwrap() - .monitored(metrics), + .monitored(metrics, config), ) } @@ -875,9 +884,9 @@ pub async fn build_remote_object_store( panic!("Passing s3-compatible is not supported, please modify the environment variable and pass in s3."); } minio if minio.starts_with("minio://") => ObjectStoreImpl::S3( - S3ObjectStore::with_minio(minio, metrics.clone()) + S3ObjectStore::with_minio(minio, metrics.clone(), config.clone()) .await - .monitored(metrics), + .monitored(metrics, config), ), "memory" => { if ident == "Meta Backup" { @@ -885,7 +894,7 @@ pub async fn build_remote_object_store( } else { tracing::warn!("You're using in-memory remote object store for {}. This should never be used in benchmarks and production environment.", ident); } - ObjectStoreImpl::InMem(InMemObjectStore::new().monitored(metrics)) + ObjectStoreImpl::InMem(InMemObjectStore::new().monitored(metrics, config)) } "memory-shared" => { if ident == "Meta Backup" { @@ -893,7 +902,11 @@ pub async fn build_remote_object_store( } else { tracing::warn!("You're using shared in-memory remote object store for {}. This should never be used in benchmarks and production environment.", ident); } - ObjectStoreImpl::InMem(InMemObjectStore::shared().monitored(metrics)) + ObjectStoreImpl::InMem(InMemObjectStore::shared().monitored(metrics, config)) + } + #[cfg(madsim)] + sim if sim.starts_with("sim://") => { + ObjectStoreImpl::Sim(SimObjectStore::new(url).monitored(metrics, config)) } other => { unimplemented!( diff --git a/src/object_store/src/object/opendal_engine/fs.rs b/src/object_store/src/object/opendal_engine/fs.rs index 23d7dcbd503e8..ece3555d5b777 100644 --- a/src/object_store/src/object/opendal_engine/fs.rs +++ b/src/object_store/src/object/opendal_engine/fs.rs @@ -17,15 +17,17 @@ use opendal::services::Fs; use opendal::Operator; use super::{EngineType, OpendalObjectStore}; +use crate::object::opendal_engine::ATOMIC_WRITE_DIR; use crate::object::ObjectResult; + impl OpendalObjectStore { /// create opendal fs engine. pub fn new_fs_engine(root: String) -> ObjectResult { // Create fs backend builder. let mut builder = Fs::default(); - builder.root(&root); - + let atomic_write_dir = format!("{}/{}", root, ATOMIC_WRITE_DIR); + builder.atomic_write_dir(&atomic_write_dir); let op: Operator = Operator::new(builder)? .layer(RetryLayer::default()) .finish(); diff --git a/src/object_store/src/object/opendal_engine/hdfs.rs b/src/object_store/src/object/opendal_engine/hdfs.rs index b52be4094df80..12ee292a85416 100644 --- a/src/object_store/src/object/opendal_engine/hdfs.rs +++ b/src/object_store/src/object/opendal_engine/hdfs.rs @@ -17,7 +17,9 @@ use opendal::services::Hdfs; use opendal::Operator; use super::{EngineType, OpendalObjectStore}; +use crate::object::opendal_engine::ATOMIC_WRITE_DIR; use crate::object::ObjectResult; + impl OpendalObjectStore { /// create opendal hdfs engine. pub fn new_hdfs_engine(namenode: String, root: String) -> ObjectResult { @@ -26,7 +28,8 @@ impl OpendalObjectStore { // Set the name node for hdfs. builder.name_node(&namenode); builder.root(&root); - + let atomic_write_dir = format!("{}/{}", root, ATOMIC_WRITE_DIR); + builder.atomic_write_dir(&atomic_write_dir); let op: Operator = Operator::new(builder)? .layer(LoggingLayer::default()) .layer(RetryLayer::default()) diff --git a/src/object_store/src/object/opendal_engine/mod.rs b/src/object_store/src/object/opendal_engine/mod.rs index 1620ee30da7d7..c1ab929d5586f 100644 --- a/src/object_store/src/object/opendal_engine/mod.rs +++ b/src/object_store/src/object/opendal_engine/mod.rs @@ -26,8 +26,11 @@ pub mod gcs; pub mod obs; -pub mod oss; - pub mod azblob; +pub mod opendal_s3; +pub mod oss; pub mod fs; + +// To make sure the the operation is consistent, we should specially set `atomic_write_dir` for fs, hdfs and webhdfs services. +const ATOMIC_WRITE_DIR: &str = "atomic_write_dir/"; diff --git a/src/object_store/src/object/opendal_engine/opendal_object_store.rs b/src/object_store/src/object/opendal_engine/opendal_object_store.rs index 9cf89c65f5581..122506d37cdfa 100644 --- a/src/object_store/src/object/opendal_engine/opendal_object_store.rs +++ b/src/object_store/src/object/opendal_engine/opendal_object_store.rs @@ -20,6 +20,7 @@ use futures::{stream, StreamExt, TryStreamExt}; use opendal::services::Memory; use opendal::{Metakey, Operator, Writer}; use risingwave_common::range::RangeBoundsExt; +use thiserror_ext::AsReport; use crate::object::{ BoxedStreamingUploader, ObjectDataStream, ObjectError, ObjectMetadata, ObjectMetadataIter, @@ -37,6 +38,7 @@ pub enum EngineType { Memory, Hdfs, Gcs, + S3, Obs, Oss, Webhdfs, @@ -116,9 +118,9 @@ impl ObjectStore for OpendalObjectStore { )); let range: Range = (range.start as u64)..(range.end as u64); let reader = self.op.reader_with(path).range(range).await?; - let stream = reader - .into_stream() - .map(|item| item.map_err(|e| ObjectError::internal(format!("OpendalError: {:?}", e)))); + let stream = reader.into_stream().map(|item| { + item.map_err(|e| ObjectError::internal(format!("OpendalError: {}", e.as_report()))) + }); Ok(Box::pin(stream)) } @@ -157,7 +159,7 @@ impl ObjectStore for OpendalObjectStore { .op .lister_with(prefix) .recursive(true) - .metakey(Metakey::ContentLength | Metakey::ContentType) + .metakey(Metakey::ContentLength) .await?; let stream = stream::unfold(object_lister, |mut object_lister| async move { @@ -189,6 +191,7 @@ impl ObjectStore for OpendalObjectStore { match self.engine_type { EngineType::Memory => "Memory", EngineType::Hdfs => "Hdfs", + EngineType::S3 => "S3", EngineType::Gcs => "Gcs", EngineType::Obs => "Obs", EngineType::Oss => "Oss", @@ -205,7 +208,11 @@ pub struct OpendalStreamingUploader { } impl OpendalStreamingUploader { pub async fn new(op: Operator, path: String) -> ObjectResult { - let writer = op.writer_with(&path).buffer(OPENDAL_BUFFER_SIZE).await?; + let writer = op + .writer_with(&path) + .concurrent(8) + .buffer(OPENDAL_BUFFER_SIZE) + .await?; Ok(Self { writer }) } } diff --git a/src/object_store/src/object/opendal_engine/opendal_s3.rs b/src/object_store/src/object/opendal_engine/opendal_s3.rs new file mode 100644 index 0000000000000..c10aff55d342b --- /dev/null +++ b/src/object_store/src/object/opendal_engine/opendal_s3.rs @@ -0,0 +1,63 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use opendal::layers::{LoggingLayer, RetryLayer}; +use opendal::services::S3; +use opendal::Operator; +use risingwave_common::config::ObjectStoreConfig; + +use super::{EngineType, OpendalObjectStore}; +use crate::object::ObjectResult; + +impl OpendalObjectStore { + /// create opendal s3 engine. + pub fn new_s3_engine( + bucket: String, + object_store_config: ObjectStoreConfig, + ) -> ObjectResult { + // Create s3 builder. + let mut builder = S3::default(); + builder.bucket(&bucket); + // For AWS S3, there is no need to set an endpoint; for other S3 compatible object stores, it is necessary to set this field. + if let Ok(endpoint_url) = std::env::var("RW_S3_ENDPOINT") { + builder.endpoint(&endpoint_url); + } + + if std::env::var("RW_IS_FORCE_PATH_STYLE").is_err() { + builder.enable_virtual_host_style(); + } + + let op: Operator = Operator::new(builder)? + .layer(LoggingLayer::default()) + .layer( + RetryLayer::new() + .with_min_delay(Duration::from_millis( + object_store_config.s3.object_store_req_retry_interval_ms, + )) + .with_max_delay(Duration::from_millis( + object_store_config.s3.object_store_req_retry_max_delay_ms, + )) + .with_max_times(object_store_config.s3.object_store_req_retry_max_attempts) + .with_factor(1.0) + .with_jitter(), + ) + .finish(); + Ok(Self { + op, + engine_type: EngineType::S3, + }) + } +} diff --git a/src/object_store/src/object/opendal_engine/webhdfs.rs b/src/object_store/src/object/opendal_engine/webhdfs.rs index ff61b39ec9e79..1f6b87b44fd5e 100644 --- a/src/object_store/src/object/opendal_engine/webhdfs.rs +++ b/src/object_store/src/object/opendal_engine/webhdfs.rs @@ -17,6 +17,7 @@ use opendal::services::Webhdfs; use opendal::Operator; use super::{EngineType, OpendalObjectStore}; +use crate::object::opendal_engine::ATOMIC_WRITE_DIR; use crate::object::ObjectResult; impl OpendalObjectStore { @@ -30,6 +31,8 @@ impl OpendalObjectStore { // NOTE: the root must be absolute path. builder.root(&root); + let atomic_write_dir = format!("{}/{}", root, ATOMIC_WRITE_DIR); + builder.atomic_write_dir(&atomic_write_dir); let op: Operator = Operator::new(builder)? .layer(LoggingLayer::default()) .layer(RetryLayer::default()) diff --git a/src/object_store/src/object/s3.rs b/src/object_store/src/object/s3.rs index 7ec004c786481..153d58fe3952d 100644 --- a/src/object_store/src/object/s3.rs +++ b/src/object_store/src/object/s3.rs @@ -36,6 +36,7 @@ use aws_smithy_runtime::client::http::hyper_014::HyperClientBuilder; use aws_smithy_runtime_api::client::http::HttpClient; use aws_smithy_runtime_api::client::result::SdkError; use aws_smithy_types::body::SdkBody; +use aws_smithy_types::error::metadata::ProvideErrorMetadata; use aws_smithy_types::retry::RetryConfig; use either::Either; use fail::fail_point; @@ -43,9 +44,10 @@ use futures::future::{try_join_all, BoxFuture, FutureExt}; use futures::{stream, Stream, StreamExt, TryStreamExt}; use hyper::Body; use itertools::Itertools; -use risingwave_common::config::ObjectStoreConfig; +use risingwave_common::config::{ObjectStoreConfig, S3ObjectStoreConfig}; use risingwave_common::monitor::connection::monitor_connector; use risingwave_common::range::RangeBoundsExt; +use thiserror_ext::AsReport; use tokio::task::JoinHandle; use tokio_retry::strategy::{jitter, ExponentialBackoff}; @@ -280,7 +282,7 @@ impl StreamingUploader for S3StreamingUploader { Ok(()) } } else if let Err(e) = self.flush_multipart_and_complete().await { - tracing::warn!("Failed to upload object {}: {:?}", self.key, e); + tracing::warn!(key = self.key, error = %e.as_report(), "Failed to upload object"); self.abort_multipart_upload().await?; Err(e) } else { @@ -384,7 +386,7 @@ impl ObjectStore for S3ObjectStore { } } }, - Self::should_retry, + RetryCondition::new(&self.config.s3), ) .await?; @@ -456,7 +458,7 @@ impl ObjectStore for S3ObjectStore { } } }, - Self::should_retry, + RetryCondition::new(&self.config.s3), ) .await?; let reader = FuturesStreamCompatByteStream::new(resp.body); @@ -530,17 +532,6 @@ impl ObjectStore for S3ObjectStore { fn store_media_type(&self) -> &'static str { "s3" } - - fn recv_buffer_size(&self) -> usize { - self.config - .s3 - .object_store_recv_buffer_size - .unwrap_or(1 << 21) - } - - fn config(&self) -> Option<&ObjectStoreConfig> { - Some(&self.config) - } } impl S3ObjectStore { @@ -628,7 +619,11 @@ impl S3ObjectStore { } /// Creates a minio client. The server should be like `minio://key:secret@address:port/bucket`. - pub async fn with_minio(server: &str, metrics: Arc) -> Self { + pub async fn with_minio( + server: &str, + metrics: Arc, + s3_object_store_config: ObjectStoreConfig, + ) -> Self { let server = server.strip_prefix("minio://").unwrap(); let (access_key_id, rest) = server.split_once(':').unwrap(); let (secret_access_key, mut rest) = rest.split_once('@').unwrap(); @@ -643,7 +638,6 @@ impl S3ObjectStore { }; let (address, bucket) = rest.split_once('/').unwrap(); - let s3_object_store_config = ObjectStoreConfig::default(); #[cfg(madsim)] let builder = aws_sdk_s3::config::Builder::new().credentials_provider( Credentials::from_keys(access_key_id, secret_access_key, None), @@ -813,32 +807,6 @@ impl S3ObjectStore { is_expiration_configured } - #[inline(always)] - fn should_retry( - err: &Either< - SdkError>, - ByteStreamError, - >, - ) -> bool { - match err { - Either::Left(err) => { - if let SdkError::DispatchFailure(e) = err { - if e.is_timeout() { - tracing::warn!(target: "http_timeout_retry", "{:?} occurs, trying to retry S3 get_object request.", e); - return true; - } - } - } - Either::Right(_) => { - // Unfortunately `ErrorKind` of `ByteStreamError` is not accessible. - // Always returns true and relies on req_retry_max_attempts to avoid infinite loop. - return true; - } - } - - false - } - #[inline(always)] fn get_retry_strategy(&self) -> impl Iterator { ExponentialBackoff::from_millis(self.config.s3.object_store_req_retry_interval_ms) @@ -849,6 +817,7 @@ impl S3ObjectStore { .map(jitter) } } + struct S3ObjectIter { buffer: VecDeque, client: Client, @@ -942,20 +911,13 @@ impl Stream for S3ObjectIter { } } -impl - From< - Either< - SdkError>, - ByteStreamError, - >, - > for ObjectError -{ - fn from( - e: Either< - SdkError>, - ByteStreamError, - >, - ) -> Self { +type RetryError = Either< + SdkError>, + ByteStreamError, +>; + +impl From for ObjectError { + fn from(e: RetryError) -> Self { match e { Either::Left(e) => e.into(), Either::Right(e) => e.into(), @@ -963,6 +925,65 @@ impl } } +struct RetryCondition { + retry_unknown_service_error: bool, + retryable_service_error_codes: Vec, +} + +impl RetryCondition { + fn new(config: &S3ObjectStoreConfig) -> Self { + Self { + retry_unknown_service_error: config.developer.object_store_retry_unknown_service_error + || config.retry_unknown_service_error, + retryable_service_error_codes: config + .developer + .object_store_retryable_service_error_codes + .clone(), + } + } +} + +impl tokio_retry::Condition for RetryCondition { + fn should_retry(&mut self, err: &RetryError) -> bool { + match err { + Either::Left(err) => match err { + SdkError::DispatchFailure(e) => { + if e.is_timeout() { + tracing::warn!(target: "http_timeout_retry", "{e:?} occurs, retry S3 get_object request."); + return true; + } + } + SdkError::ServiceError(e) => match e.err().code() { + None => { + if self.retry_unknown_service_error { + tracing::warn!(target: "unknown_service_error", "{e:?} occurs, retry S3 get_object request."); + return true; + } + } + Some(code) => { + if self + .retryable_service_error_codes + .iter() + .any(|s| s.as_str().eq_ignore_ascii_case(code)) + { + tracing::warn!(target: "retryable_service_error", "{e:?} occurs, retry S3 get_object request."); + return true; + } + } + }, + _ => {} + }, + Either::Right(_) => { + // Unfortunately `ErrorKind` of `ByteStreamError` is not accessible. + // Always returns true and relies on req_retry_max_attempts to avoid infinite loop. + return true; + } + } + + false + } +} + #[cfg(test)] #[cfg(not(madsim))] mod tests { diff --git a/src/object_store/src/object/sim/client.rs b/src/object_store/src/object/sim/client.rs new file mode 100644 index 0000000000000..4e7aa67834a59 --- /dev/null +++ b/src/object_store/src/object/sim/client.rs @@ -0,0 +1,48 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::fmt::Debug; +use std::net::SocketAddr; + +use madsim::net::{Endpoint, Payload}; + +use super::service::{Request, Response}; +use crate::object::error::ObjectResult as Result; + +#[derive(Debug, Clone)] +pub struct Client { + addr: SocketAddr, +} + +impl Client { + pub fn new(addr: SocketAddr) -> Self { + Self { addr } + } + + pub(crate) async fn send_request(&self, req: Request) -> Result { + let resp = self.send_request_io(req).await?; + let resp = *resp + .downcast::>() + .expect("failed to downcast"); + resp + } + + async fn send_request_io(&self, req: Request) -> std::io::Result { + let addr = self.addr; + let ep = Endpoint::connect(addr).await?; + let (tx, mut rx) = ep.connect1(addr).await?; + tx.send(Box::new(req)).await?; + rx.recv().await + } +} diff --git a/src/cmd/src/bin/compute_node.rs b/src/object_store/src/object/sim/error.rs similarity index 52% rename from src/cmd/src/bin/compute_node.rs rename to src/object_store/src/object/sim/error.rs index 1537a0169c3e4..29ca09f6825b7 100644 --- a/src/cmd/src/bin/compute_node.rs +++ b/src/object_store/src/object/sim/error.rs @@ -12,6 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. -#![cfg_attr(coverage, feature(coverage_attribute))] +use thiserror::Error; -risingwave_cmd::main!(compute); +#[derive(Error, Debug)] +pub enum SimError { + #[error("NotFound error: {0}")] + NotFound(String), + #[error("Other error: {0}")] + Other(String), +} + +impl SimError { + pub fn is_object_not_found_error(&self) -> bool { + matches!(self, SimError::NotFound(_)) + } +} + +impl SimError { + pub(crate) fn not_found(msg: impl ToString) -> Self { + SimError::NotFound(msg.to_string()) + } + + pub(crate) fn other(msg: impl ToString) -> Self { + SimError::Other(msg.to_string()) + } +} diff --git a/src/object_store/src/object/sim/mod.rs b/src/object_store/src/object/sim/mod.rs new file mode 100644 index 0000000000000..31f34cbcd3267 --- /dev/null +++ b/src/object_store/src/object/sim/mod.rs @@ -0,0 +1,254 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +mod client; +mod error; +use bytes::{BufMut, BytesMut}; +pub use error::SimError; +use futures::Stream; +mod rpc_server; +pub use rpc_server::SimServer; +mod service; + +use std::net::SocketAddr; +use std::ops::{Range, RangeBounds}; +use std::pin::Pin; +use std::task::{Context, Poll}; + +use madsim::rand::{thread_rng, RngCore}; +use madsim::time::{sleep, Duration}; +use risingwave_common::range::RangeBoundsExt; + +use self::client::Client; +use self::service::Response; +use super::{ + BoxedStreamingUploader, Bytes, ObjectDataStream, ObjectError, ObjectMetadata, + ObjectMetadataIter, ObjectRangeBounds, ObjectResult, ObjectStore, StreamingUploader, +}; + +pub struct SimStreamingUploader { + client: Client, + key: String, + buf: BytesMut, +} + +impl SimStreamingUploader { + fn new(client: Client, key: String) -> Self { + Self { + client, + key, + buf: Default::default(), + } + } +} + +#[async_trait::async_trait] +impl StreamingUploader for SimStreamingUploader { + async fn write_bytes(&mut self, data: Bytes) -> ObjectResult<()> { + self.buf.put(data); + Ok(()) + } + + async fn finish(mut self: Box) -> ObjectResult<()> { + if self.buf.is_empty() { + Err(ObjectError::internal("upload empty object")) + } else { + let resp = self + .client + .send_request(service::Request::Upload { + path: self.key, + obj: self.buf.freeze(), + }) + .await?; + let Response::Upload = resp else { + panic!("expect Response::Upload"); + }; + Ok(()) + } + } + + fn get_memory_usage(&self) -> u64 { + self.buf.len() as u64 + } +} + +pub struct SimDataIterator { + data: Bytes, + offset: usize, +} + +impl SimDataIterator { + pub fn new(data: Bytes) -> Self { + Self { data, offset: 0 } + } +} + +impl Stream for SimDataIterator { + type Item = ObjectResult; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + const MAX_PACKET_SIZE: usize = 128 * 1024; + if self.offset >= self.data.len() { + return Poll::Ready(None); + } + let read_len = std::cmp::min(self.data.len() - self.offset, MAX_PACKET_SIZE); + let data = self.data.slice(self.offset..(self.offset + read_len)); + self.offset += read_len; + Poll::Ready(Some(Ok(data))) + } +} + +pub struct SimObjectStore { + client: Client, +} + +#[async_trait::async_trait] +impl ObjectStore for SimObjectStore { + fn get_object_prefix(&self, _obj_id: u64) -> String { + String::default() + } + + async fn upload(&self, path: &str, obj: Bytes) -> ObjectResult<()> { + if obj.is_empty() { + Err(ObjectError::internal("upload empty object")) + } else { + let path = path.to_string(); + let resp = self + .client + .send_request(service::Request::Upload { path, obj }) + .await?; + if let Response::Upload = resp { + Ok(()) + } else { + panic!("expect Response::Upload"); + } + } + } + + async fn streaming_upload(&self, path: &str) -> ObjectResult { + Ok(Box::new(SimStreamingUploader::new( + self.client.clone(), + path.to_string(), + ))) + } + + async fn read(&self, path: &str, range: impl ObjectRangeBounds) -> ObjectResult { + let path = path.to_string(); + let resp = self + .client + .send_request(service::Request::Read { path }) + .await?; + if let Response::Read(obj) = resp { + if let Some(end) = range.end() + && end > obj.len() + { + return Err(SimError::other("bad block offset and size").into()); + } + Ok(obj.slice(range)) + } else { + panic!("expect Response::Read"); + } + } + + async fn metadata(&self, path: &str) -> ObjectResult { + let path = path.to_string(); + if let Response::Metadata(m) = self + .client + .send_request(service::Request::Metadata { path }) + .await? + { + Ok(m) + } else { + panic!("expect Response::Metadata"); + } + } + + async fn streaming_read( + &self, + path: &str, + range: Range, + ) -> ObjectResult { + let path = path.to_string(); + let resp = self + .client + .send_request(service::Request::Read { path }) + .await?; + let Response::Read(body) = resp else { + panic!("expect Response::Read"); + }; + + Ok(Box::pin(SimDataIterator::new(body.slice(range)))) + } + + async fn delete(&self, path: &str) -> ObjectResult<()> { + let path = path.to_string(); + let resp = self + .client + .send_request(service::Request::Delete { path }) + .await?; + if let Response::Delete = resp { + Ok(()) + } else { + panic!("expect Response::Delete"); + } + } + + async fn delete_objects(&self, paths: &[String]) -> ObjectResult<()> { + let resp = self + .client + .send_request(service::Request::DeleteObjects { + paths: paths.to_vec(), + }) + .await?; + if let Response::DeleteObjects = resp { + Ok(()) + } else { + panic!("expect Response::DeleteObjects"); + } + } + + async fn list(&self, path: &str) -> ObjectResult { + let path = path.to_string(); + let resp = self + .client + .send_request(service::Request::List { path }) + .await?; + if let Response::List(o) = resp { + Ok(Box::pin(o)) + } else { + panic!("expect Response::List"); + } + } + + fn store_media_type(&self) -> &'static str { + "sim" + } +} + +impl SimObjectStore { + pub fn new(addr: &str) -> Self { + let addr = addr.strip_prefix("sim://").unwrap(); + let (_access_key_id, rest) = addr.split_once(':').unwrap(); + let (_secret_access_key, rest) = rest.split_once('@').unwrap(); + let (address, _bucket) = rest.split_once('/').unwrap(); + + Self { + client: Client::new( + address + .parse::() + .expect(&format!("parse SockAddr failed: {}", address)), + ), + } + } +} diff --git a/src/object_store/src/object/sim/rpc_server.rs b/src/object_store/src/object/sim/rpc_server.rs new file mode 100644 index 0000000000000..068dbed5a27a4 --- /dev/null +++ b/src/object_store/src/object/sim/rpc_server.rs @@ -0,0 +1,57 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::io::Result; +use std::net::SocketAddr; +use std::sync::Arc; + +use madsim::net::{Endpoint, Payload}; + +use super::service::{Request, SimService}; + +/// A simulated ObjectStore server. +#[derive(Default, Clone)] +pub struct SimServer {} + +impl SimServer { + pub fn builder() -> Self { + SimServer::default() + } + + pub async fn serve(self, addr: SocketAddr) -> Result<()> { + let ep = Endpoint::bind(addr).await?; + let service = SimService::new(); + let service = Arc::new(service); + + loop { + let (tx, mut rx, _) = ep.accept1().await?; + let service = service.clone(); + madsim::task::spawn(async move { + let request = *rx.recv().await?.downcast::().unwrap(); + use super::service::Request::*; + + let response: Payload = match request { + Upload { path, obj } => Box::new(service.upload(path, obj).await), + Read { path } => Box::new(service.read(path).await), + Delete { path } => Box::new(service.delete(path).await), + DeleteObjects { paths } => Box::new(service.delete_objects(paths).await), + List { path } => Box::new(service.list(path).await), + Metadata { path } => Box::new(service.metadata(path).await), + }; + tx.send(response).await?; + Ok(()) as Result<()> + }); + } + } +} diff --git a/src/object_store/src/object/sim/service.rs b/src/object_store/src/object/sim/service.rs new file mode 100644 index 0000000000000..fb03a1b13c173 --- /dev/null +++ b/src/object_store/src/object/sim/service.rs @@ -0,0 +1,151 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::{BTreeMap, VecDeque}; +use std::pin::Pin; +use std::task::{Context, Poll}; +use std::time::{SystemTime, UNIX_EPOCH}; + +use bytes::Bytes; +use futures::Stream; +use itertools::Itertools; +use spin::Mutex; + +use crate::object::error::ObjectResult as Result; +use crate::object::sim::SimError; +use crate::object::{ObjectError, ObjectMetadata}; + +#[derive(Debug)] +pub(crate) enum Request { + Upload { path: String, obj: Bytes }, + Read { path: String }, + Delete { path: String }, + DeleteObjects { paths: Vec }, + List { path: String }, + Metadata { path: String }, +} + +#[derive(Debug)] +pub(crate) enum Response { + Upload, + Read(Bytes), + Delete, + DeleteObjects, + List(SimObjectIter), + Metadata(ObjectMetadata), +} + +#[derive(Debug)] +pub(crate) struct SimService { + storage: Mutex>, +} + +fn get_obj_meta(path: &str, obj: &Bytes) -> Result { + Ok(ObjectMetadata { + key: path.to_string(), + last_modified: SystemTime::now() + .duration_since(UNIX_EPOCH) + .map_err(ObjectError::internal)? + .as_secs_f64(), + total_size: obj.len(), + }) +} + +impl SimService { + pub fn new() -> Self { + SimService { + storage: Mutex::new(BTreeMap::new()), + } + } + + pub async fn upload(&self, path: String, obj: Bytes) -> Result { + let metadata = get_obj_meta(&path, &obj)?; + self.storage.lock().insert(path.into(), (metadata, obj)); + Ok(Response::Upload) + } + + pub async fn read(&self, path: String) -> Result { + let storage = self.storage.lock(); + let obj = storage + .get(&path) + .map(|(_, o)| o) + .ok_or_else(|| SimError::NotFound(format!("no object at path '{}'", path)))?; + Ok(Response::Read(obj.clone())) + } + + pub async fn delete(&self, path: String) -> Result { + self.storage.lock().remove(&path); + Ok(Response::Delete) + } + + pub async fn delete_objects(&self, paths: Vec) -> Result { + for path in paths { + self.storage.lock().remove(&path); + } + Ok(Response::DeleteObjects) + } + + pub async fn list(&self, prefix: String) -> Result { + if prefix.is_empty() { + return Ok(Response::List(SimObjectIter::new(vec![]))); + } + + let mut scan_end = prefix.chars().collect_vec(); + let next = *scan_end.last().unwrap() as u8 + 1; + *scan_end.last_mut().unwrap() = next as char; + let scan_end = scan_end.into_iter().collect::(); + + let list_result = self + .storage + .lock() + .range(prefix..scan_end) + .map(|(_, (o, _))| o.clone()) + .collect_vec(); + Ok(Response::List(SimObjectIter::new(list_result))) + } + + pub async fn metadata(&self, path: String) -> Result { + self.storage + .lock() + .get(&path) + .map(|(metadata, _)| metadata) + .cloned() + .ok_or_else(|| SimError::not_found(format!("no object at path '{}'", path)).into()) + .map(|meta| Response::Metadata(meta)) + } +} + +#[derive(Debug)] +pub(crate) struct SimObjectIter { + list_result: VecDeque, +} + +impl SimObjectIter { + fn new(list_result: Vec) -> Self { + Self { + list_result: list_result.into(), + } + } +} + +impl Stream for SimObjectIter { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, _cx: &mut Context<'_>) -> Poll> { + if let Some(i) = self.list_result.pop_front() { + return Poll::Ready(Some(Ok(i))); + } + Poll::Ready(None) + } +} diff --git a/src/prost/build.rs b/src/prost/build.rs index 7b06a5e5fb7e5..cd65b20d6f192 100644 --- a/src/prost/build.rs +++ b/src/prost/build.rs @@ -51,6 +51,7 @@ fn main() -> Result<(), Box> { "stream_plan", "stream_service", "task_service", + "telemetry", "user", ]; let protos: Vec = proto_files @@ -115,6 +116,28 @@ fn main() -> Result<(), Box> { .type_attribute("plan_common.Cardinality", "#[derive(Eq, Hash, Copy)]") .type_attribute("plan_common.ExternalTableDesc", "#[derive(Eq, Hash)]") .type_attribute("plan_common.ColumnDesc", "#[derive(Eq, Hash)]") + .type_attribute("plan_common.AdditionalColumn", "#[derive(Eq, Hash)]") + .type_attribute( + "plan_common.AdditionalColumn.column_type", + "#[derive(Eq, Hash)]", + ) + .type_attribute("plan_common.AdditionalColumnNormal", "#[derive(Eq, Hash)]") + .type_attribute("plan_common.AdditionalColumnKey", "#[derive(Eq, Hash)]") + .type_attribute( + "plan_common.AdditionalColumnPartition", + "#[derive(Eq, Hash)]", + ) + .type_attribute( + "plan_common.AdditionalColumnTimestamp", + "#[derive(Eq, Hash)]", + ) + .type_attribute( + "plan_common.AdditionalColumnFilename", + "#[derive(Eq, Hash)]", + ) + .type_attribute("plan_common.AdditionalColumnHeader", "#[derive(Eq, Hash)]") + .type_attribute("plan_common.AdditionalColumnHeaders", "#[derive(Eq, Hash)]") + .type_attribute("plan_common.AdditionalColumnOffset", "#[derive(Eq, Hash)]") .type_attribute("common.ColumnOrder", "#[derive(Eq, Hash)]") .type_attribute("common.OrderType", "#[derive(Eq, Hash)]") .type_attribute("common.Buffer", "#[derive(Eq)]") diff --git a/src/prost/src/lib.rs b/src/prost/src/lib.rs index 0a503ff99017e..82f399d1a3de6 100644 --- a/src/prost/src/lib.rs +++ b/src/prost/src/lib.rs @@ -89,6 +89,9 @@ pub mod java_binding; #[cfg_attr(madsim, path = "sim/health.rs")] pub mod health; #[rustfmt::skip] +#[path = "sim/telemetry.rs"] +pub mod telemetry; +#[rustfmt::skip] #[path = "connector_service.serde.rs"] pub mod connector_service_serde; #[rustfmt::skip] @@ -151,6 +154,9 @@ pub mod backup_service_serde; #[rustfmt::skip] #[path = "java_binding.serde.rs"] pub mod java_binding_serde; +#[rustfmt::skip] +#[path = "telemetry.serde.rs"] +pub mod telemetry_serde; #[derive(Clone, PartialEq, Eq, Debug, Error)] #[error("field `{0}` not found")] @@ -170,6 +176,26 @@ impl FromStr for crate::expr::table_function::PbType { } } +impl stream_plan::MaterializeNode { + pub fn dist_key_indices(&self) -> Vec { + self.get_table() + .unwrap() + .distribution_key + .iter() + .map(|i| *i as u32) + .collect() + } + + pub fn column_ids(&self) -> Vec { + self.get_table() + .unwrap() + .columns + .iter() + .map(|c| c.get_column_desc().unwrap().column_id) + .collect() + } +} + #[cfg(test)] mod tests { use crate::data::{data_type, DataType}; diff --git a/src/risedevtool/Cargo.toml b/src/risedevtool/Cargo.toml index 3b4acc0f9877e..3fbea69fc9445 100644 --- a/src/risedevtool/Cargo.toml +++ b/src/risedevtool/Cargo.toml @@ -20,7 +20,7 @@ clap = { version = "4", features = ["derive"] } console = "0.15" fs-err = "2.11.0" glob = "0.3" -google-cloud-pubsub = "0.22" +google-cloud-pubsub = "0.23" indicatif = "0.17" itertools = "0.12" rdkafka = { workspace = true } @@ -32,6 +32,7 @@ serde_json = "1" serde_with = "3" serde_yaml = "0.9" tempfile = "3" +thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = [ "rt", "rt-multi-thread", diff --git a/src/risedevtool/config/Cargo.toml b/src/risedevtool/config/Cargo.toml index 7a115f4b2ec11..3b9972c82916b 100644 --- a/src/risedevtool/config/Cargo.toml +++ b/src/risedevtool/config/Cargo.toml @@ -12,7 +12,7 @@ anyhow = { version = "1", features = ["backtrace"] } clap = { version = "4", features = ["derive"] } console = "0.15" dialoguer = "0.11" -enum-iterator = "1" +enum-iterator = "2" fs-err = "2.11.0" itertools = "0.12" diff --git a/src/risedevtool/config/src/main.rs b/src/risedevtool/config/src/main.rs index 292c191a8bf84..e8874456bb50b 100644 --- a/src/risedevtool/config/src/main.rs +++ b/src/risedevtool/config/src/main.rs @@ -70,7 +70,6 @@ pub enum Components { BuildConnectorNode, Dashboard, Release, - AllInOne, Sanitizer, DynamicLinking, HummockTrace, @@ -92,7 +91,6 @@ impl Components { Self::Dashboard => "[Build] Dashboard", Self::Tracing => "[Component] Tracing: Grafana Tempo", Self::Release => "[Build] Enable release mode", - Self::AllInOne => "[Build] Enable all-in-one binary", Self::Sanitizer => "[Build] Enable sanitizer", Self::DynamicLinking => "[Build] Enable dynamic linking", Self::HummockTrace => "[Build] Hummock Trace", @@ -148,12 +146,6 @@ you download Grafana Tempo." Self::Release => { " Build RisingWave in release mode" - } - Self::AllInOne => { - " -With this option enabled, RiseDev will help you create -symlinks to `risingwave` all-in-one binary, so as to build -and use `risingwave` in all-in-one mode." } Self::Sanitizer => { " @@ -212,7 +204,6 @@ As a result, RisingWave will dump the core on panics. "ENABLE_COMPUTE_TRACING" => Some(Self::Tracing), "ENABLE_RELEASE_PROFILE" => Some(Self::Release), "ENABLE_DYNAMIC_LINKING" => Some(Self::DynamicLinking), - "ENABLE_ALL_IN_ONE" => Some(Self::AllInOne), "ENABLE_SANITIZER" => Some(Self::Sanitizer), "ENABLE_REDIS" => Some(Self::Redis), "ENABLE_BUILD_RW_CONNECTOR" => Some(Self::BuildConnectorNode), @@ -234,7 +225,6 @@ As a result, RisingWave will dump the core on panics. Self::Dashboard => "ENABLE_BUILD_DASHBOARD", Self::Tracing => "ENABLE_COMPUTE_TRACING", Self::Release => "ENABLE_RELEASE_PROFILE", - Self::AllInOne => "ENABLE_ALL_IN_ONE", Self::Sanitizer => "ENABLE_SANITIZER", Self::BuildConnectorNode => "ENABLE_BUILD_RW_CONNECTOR", Self::DynamicLinking => "ENABLE_DYNAMIC_LINKING", diff --git a/src/risedevtool/src/bin/risedev-dev.rs b/src/risedevtool/src/bin/risedev-dev.rs index ea9099eae319c..9723ee89fbd51 100644 --- a/src/risedevtool/src/bin/risedev-dev.rs +++ b/src/risedevtool/src/bin/risedev-dev.rs @@ -31,6 +31,7 @@ use risedev::{ RISEDEV_SESSION_NAME, }; use tempfile::tempdir; +use thiserror_ext::AsReport; use yaml_rust::YamlEmitter; #[derive(Default)] @@ -444,9 +445,9 @@ fn main() -> Result<()> { } Err(err) => { println!( - "{} - Failed to start: {:?}", // with `Caused by` + "{} - Failed to start: {:#}", // pretty with `Caused by` style("ERROR").red().bold(), - err, + err.as_report(), ); println!(); println!( diff --git a/src/risedevtool/src/bin/risedev-docslt.rs b/src/risedevtool/src/bin/risedev-docslt.rs index d073ff40f3125..627e7b028861f 100644 --- a/src/risedevtool/src/bin/risedev-docslt.rs +++ b/src/risedevtool/src/bin/risedev-docslt.rs @@ -17,6 +17,7 @@ use std::path::{Path, PathBuf}; use anyhow::Result; use itertools::Itertools; +use thiserror_ext::AsReport; use tracing::*; #[derive(Clone, PartialEq, Eq, PartialOrd, Ord)] @@ -89,7 +90,7 @@ fn main() -> Result<()> { let path = match entry { Ok(path) => path, Err(e) => { - error!("{:?}", e); + error!("{}", e.as_report()); continue; } }; diff --git a/src/risedevtool/src/preflight_check.rs b/src/risedevtool/src/preflight_check.rs index 17dc48884fea8..9b25d39423566 100644 --- a/src/risedevtool/src/preflight_check.rs +++ b/src/risedevtool/src/preflight_check.rs @@ -17,6 +17,7 @@ use std::process::Command; use anyhow::Result; use console::style; +use thiserror_ext::AsReport; fn preflight_check_proxy() -> Result<()> { if env::var("http_proxy").is_ok() @@ -72,7 +73,7 @@ pub fn preflight_check() -> Result<()> { "[{}] {} - failed to run proxy preflight check: {}", style("risedev-preflight-check").bold(), style("WARN").yellow().bold(), - e + e.as_report() ); } @@ -81,7 +82,7 @@ pub fn preflight_check() -> Result<()> { "[{}] {} - failed to run ulimit preflight check: {}", style("risedev-preflight-check").bold(), style("WARN").yellow().bold(), - e + e.as_report() ); } diff --git a/src/risedevtool/src/service_config.rs b/src/risedevtool/src/service_config.rs index a254ce80a8326..0d61f9a873afa 100644 --- a/src/risedevtool/src/service_config.rs +++ b/src/risedevtool/src/service_config.rs @@ -140,6 +140,10 @@ pub struct MinioConfig { pub hummock_bucket: String, pub provide_prometheus: Option>, + + // For rate limiting minio in a test environment. + pub api_requests_max: usize, + pub api_requests_deadline: String, } #[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] diff --git a/src/risedevtool/src/task/compactor_service.rs b/src/risedevtool/src/task/compactor_service.rs index 393a40d5b1aa5..1a2b24b36bc97 100644 --- a/src/risedevtool/src/task/compactor_service.rs +++ b/src/risedevtool/src/task/compactor_service.rs @@ -34,15 +34,9 @@ impl CompactorService { fn compactor(&self) -> Result { let prefix_bin = env::var("PREFIX_BIN")?; - if let Ok(x) = env::var("ENABLE_ALL_IN_ONE") - && x == "true" - { - Ok(Command::new( - Path::new(&prefix_bin).join("risingwave").join("compactor"), - )) - } else { - Ok(Command::new(Path::new(&prefix_bin).join("compactor"))) - } + Ok(Command::new( + Path::new(&prefix_bin).join("risingwave").join("compactor"), + )) } /// Apply command args according to config diff --git a/src/risedevtool/src/task/compute_node_service.rs b/src/risedevtool/src/task/compute_node_service.rs index b8fc7c1184dfe..3dfba5f3e3e16 100644 --- a/src/risedevtool/src/task/compute_node_service.rs +++ b/src/risedevtool/src/task/compute_node_service.rs @@ -34,17 +34,11 @@ impl ComputeNodeService { fn compute_node(&self) -> Result { let prefix_bin = env::var("PREFIX_BIN")?; - if let Ok(x) = env::var("ENABLE_ALL_IN_ONE") - && x == "true" - { - Ok(Command::new( - Path::new(&prefix_bin) - .join("risingwave") - .join("compute-node"), - )) - } else { - Ok(Command::new(Path::new(&prefix_bin).join("compute-node"))) - } + Ok(Command::new( + Path::new(&prefix_bin) + .join("risingwave") + .join("compute-node"), + )) } /// Apply command args according to config diff --git a/src/risedevtool/src/task/frontend_service.rs b/src/risedevtool/src/task/frontend_service.rs index e37349dafdc1d..2ce67db6b7582 100644 --- a/src/risedevtool/src/task/frontend_service.rs +++ b/src/risedevtool/src/task/frontend_service.rs @@ -35,17 +35,11 @@ impl FrontendService { fn frontend(&self) -> Result { let prefix_bin = env::var("PREFIX_BIN")?; - if let Ok(x) = env::var("ENABLE_ALL_IN_ONE") - && x == "true" - { - Ok(Command::new( - Path::new(&prefix_bin) - .join("risingwave") - .join("frontend-node"), - )) - } else { - Ok(Command::new(Path::new(&prefix_bin).join("frontend"))) - } + Ok(Command::new( + Path::new(&prefix_bin) + .join("risingwave") + .join("frontend-node"), + )) } /// Apply command args according to config diff --git a/src/risedevtool/src/task/meta_node_service.rs b/src/risedevtool/src/task/meta_node_service.rs index 83b55c0d7104f..d88580618763a 100644 --- a/src/risedevtool/src/task/meta_node_service.rs +++ b/src/risedevtool/src/task/meta_node_service.rs @@ -35,15 +35,9 @@ impl MetaNodeService { fn meta_node(&self) -> Result { let prefix_bin = env::var("PREFIX_BIN")?; - if let Ok(x) = env::var("ENABLE_ALL_IN_ONE") - && x == "true" - { - Ok(Command::new( - Path::new(&prefix_bin).join("risingwave").join("meta-node"), - )) - } else { - Ok(Command::new(Path::new(&prefix_bin).join("meta-node"))) - } + Ok(Command::new( + Path::new(&prefix_bin).join("risingwave").join("meta-node"), + )) } /// Apply command args according to config diff --git a/src/risedevtool/src/task/minio_service.rs b/src/risedevtool/src/task/minio_service.rs index 62f6718f66aed..4c595a96f198e 100644 --- a/src/risedevtool/src/task/minio_service.rs +++ b/src/risedevtool/src/task/minio_service.rs @@ -53,6 +53,17 @@ impl MinioService { // https://github.com/risingwavelabs/risingwave/pull/3012 // https://docs.min.io/minio/baremetal/installation/deploy-minio-single-node-single-drive.html#id3 .env("MINIO_CI_CD", "1"); + if config.api_requests_max > 0 { + // Rate limit minio + cmd.env( + "MINIO_API_REQUESTS_MAX", + config.api_requests_max.to_string(), + ); + } + if !config.api_requests_deadline.is_empty() { + // Rate limit minio + cmd.env("MINIO_API_REQUESTS_DEADLINE", &config.api_requests_deadline); + } let provide_prometheus = config.provide_prometheus.as_ref().unwrap(); match provide_prometheus.len() { diff --git a/src/rpc_client/Cargo.toml b/src/rpc_client/Cargo.toml index 7445de59ff9c7..4d84cff599d2d 100644 --- a/src/rpc_client/Cargo.toml +++ b/src/rpc_client/Cargo.toml @@ -17,7 +17,7 @@ normal = ["workspace-hack"] anyhow = "1" async-trait = "0.1" easy-ext = "1" -either = "1.9.0" +either = "1.10.0" futures = { version = "0.3", default-features = false, features = ["alloc"] } http = "0.2" hyper = "0.14" @@ -29,6 +29,7 @@ risingwave_common = { workspace = true } risingwave_error = { workspace = true } risingwave_hummock_sdk = { workspace = true } risingwave_pb = { workspace = true } +rw_futures_util = { workspace = true } static_assertions = "1" thiserror = "1" thiserror-ext = { workspace = true } diff --git a/src/rpc_client/src/compute_client.rs b/src/rpc_client/src/compute_client.rs index 9d37625b20eb4..8e96f7a81702d 100644 --- a/src/rpc_client/src/compute_client.rs +++ b/src/rpc_client/src/compute_client.rs @@ -27,9 +27,10 @@ use risingwave_pb::compute::config_service_client::ConfigServiceClient; use risingwave_pb::compute::{ShowConfigRequest, ShowConfigResponse}; use risingwave_pb::monitor_service::monitor_service_client::MonitorServiceClient; use risingwave_pb::monitor_service::{ - AnalyzeHeapRequest, AnalyzeHeapResponse, HeapProfilingRequest, HeapProfilingResponse, - ListHeapProfilingRequest, ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, - StackTraceRequest, StackTraceResponse, + AnalyzeHeapRequest, AnalyzeHeapResponse, GetBackPressureRequest, GetBackPressureResponse, + HeapProfilingRequest, HeapProfilingResponse, ListHeapProfilingRequest, + ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, StackTraceRequest, + StackTraceResponse, }; use risingwave_pb::plan_common::ExprContext; use risingwave_pb::task_service::exchange_service_client::ExchangeServiceClient; @@ -197,6 +198,15 @@ impl ComputeClient { .into_inner()) } + pub async fn get_back_pressure(&self) -> Result { + Ok(self + .monitor_client + .to_owned() + .get_back_pressure(GetBackPressureRequest::default()) + .await? + .into_inner()) + } + pub async fn profile(&self, sleep_s: u64) -> Result { Ok(self .monitor_client diff --git a/src/rpc_client/src/connector_client.rs b/src/rpc_client/src/connector_client.rs index cd53cd019ea64..d627a692735c3 100644 --- a/src/rpc_client/src/connector_client.rs +++ b/src/rpc_client/src/connector_client.rs @@ -16,7 +16,7 @@ use std::collections::HashMap; use std::fmt::Debug; use std::time::Duration; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use futures::TryStreamExt; use risingwave_common::config::{MAX_CONNECTION_WINDOW_SIZE, STREAM_WINDOW_SIZE}; use risingwave_common::monitor::connection::{EndpointExt, TcpConfig}; @@ -30,6 +30,7 @@ use risingwave_pb::connector_service::sink_writer_stream_request::{ }; use risingwave_pb::connector_service::sink_writer_stream_response::CommitResponse; use risingwave_pb::connector_service::*; +use thiserror_ext::AsReport; use tokio_stream::wrappers::ReceiverStream; use tonic::transport::{Channel, Endpoint}; use tonic::Streaming; @@ -150,8 +151,9 @@ impl ConnectorClient { Ok(client) => Some(client), Err(e) => { error!( - "invalid connector endpoint {:?}: {:?}", - connector_endpoint, e + endpoint = connector_endpoint, + error = %e.as_report(), + "invalid connector endpoint", ); None } @@ -162,12 +164,7 @@ impl ConnectorClient { #[allow(clippy::unused_async)] pub async fn new(connector_endpoint: &String) -> Result { let endpoint = Endpoint::from_shared(format!("http://{}", connector_endpoint)) - .map_err(|e| { - RpcError::Internal(anyhow!(format!( - "invalid connector endpoint `{}`: {:?}", - &connector_endpoint, e - ))) - })? + .with_context(|| format!("invalid connector endpoint `{}`", connector_endpoint))? .initial_connection_window_size(MAX_CONNECTION_WINDOW_SIZE) .initial_stream_window_size(STREAM_WINDOW_SIZE) .connect_timeout(Duration::from_secs(5)); diff --git a/src/rpc_client/src/error.rs b/src/rpc_client/src/error.rs index 92c9ebe5bf338..e44d3ac8c0b9f 100644 --- a/src/rpc_client/src/error.rs +++ b/src/rpc_client/src/error.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::{ErrorCode, RwError}; use risingwave_common::util::meta_addr::MetaAddressStrategyParseError; use thiserror::Error; @@ -54,12 +53,3 @@ impl From for RpcError { RpcError::GrpcStatus(Box::new(TonicStatusWrapper::new(s))) } } - -impl From for RwError { - fn from(r: RpcError) -> Self { - match r { - RpcError::GrpcStatus(status) => TonicStatusWrapper::into(*status), - _ => ErrorCode::RpcError(r.into()).into(), - } - } -} diff --git a/src/rpc_client/src/lib.rs b/src/rpc_client/src/lib.rs index 726d2e4c6c986..0485465499f5a 100644 --- a/src/rpc_client/src/lib.rs +++ b/src/rpc_client/src/lib.rs @@ -33,7 +33,7 @@ use std::future::Future; use std::iter::repeat; use std::sync::Arc; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use futures::future::try_join_all; use futures::stream::{BoxStream, Peekable}; @@ -46,7 +46,7 @@ use risingwave_pb::meta::heartbeat_request::extra_info; use tokio::sync::mpsc::{channel, Receiver, Sender}; pub mod error; -use error::{Result, RpcError}; +use error::Result; mod compactor_client; mod compute_client; mod connector_client; @@ -61,7 +61,7 @@ pub use compute_client::{ComputeClient, ComputeClientPool, ComputeClientPoolRef} pub use connector_client::{ConnectorClient, SinkCoordinatorStreamHandle, SinkWriterStreamHandle}; pub use hummock_meta_client::{CompactionEventItem, HummockMetaClient}; pub use meta_client::{MetaClient, SinkCoordinationRpcClient}; -use risingwave_common::util::await_future_with_monitor_error_stream; +use rw_futures_util::await_future_with_monitor_error_stream; pub use sink_coordinate_client::CoordinatorStreamHandle; pub use stream_client::{StreamClient, StreamClientPool, StreamClientPoolRef}; @@ -120,9 +120,7 @@ where S::new_clients(addr.clone(), self.connection_pool_size as usize), ) .await - .map_err(|e| -> RpcError { - anyhow!("failed to create RPC client to {addr}: {:?}", e).into() - })? + .with_context(|| format!("failed to create RPC client to {addr}"))? .choose(&mut rand::thread_rng()) .unwrap() .clone()) diff --git a/src/rpc_client/src/meta_client.rs b/src/rpc_client/src/meta_client.rs index 2dc51961c901e..4d0b3b6b7a673 100644 --- a/src/rpc_client/src/meta_client.rs +++ b/src/rpc_client/src/meta_client.rs @@ -19,7 +19,7 @@ use std::sync::Arc; use std::thread; use std::time::{Duration, SystemTime}; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; use async_trait::async_trait; use either::Either; use futures::stream::BoxStream; @@ -86,6 +86,7 @@ use risingwave_pb::stream_plan::StreamFragmentGraph; use risingwave_pb::user::update_user_request::UpdateField; use risingwave_pb::user::user_service_client::UserServiceClient; use risingwave_pb::user::*; +use thiserror_ext::AsReport; use tokio::sync::mpsc::{unbounded_channel, Receiver, UnboundedSender}; use tokio::sync::oneshot::Sender; use tokio::sync::{mpsc, oneshot, RwLock}; @@ -457,14 +458,24 @@ impl MetaClient { Ok(resp.version) } + pub async fn alter_source_with_sr(&self, source: PbSource) -> Result { + let request = AlterSourceRequest { + source: Some(source), + }; + let resp = self.inner.alter_source(request).await?; + Ok(resp.version) + } + pub async fn alter_parallelism( &self, table_id: u32, parallelism: PbTableParallelism, + deferred: bool, ) -> Result<()> { let request = AlterParallelismRequest { table_id, parallelism: Some(parallelism), + deferred, }; self.inner.alter_parallelism(request).await?; @@ -731,10 +742,10 @@ impl MetaClient { { Ok(Ok(_)) => {} Ok(Err(err)) => { - tracing::warn!("Failed to send_heartbeat: error {}", err); + tracing::warn!(error = %err.as_report(), "Failed to send_heartbeat"); } - Err(err) => { - tracing::warn!("Failed to send_heartbeat: timeout {}", err); + Err(_) => { + tracing::warn!("Failed to send_heartbeat: timeout"); } } } @@ -1414,7 +1425,7 @@ impl HummockMetaClient for MetaClient { .expect("Clock may have gone backwards") .as_millis() as u64, }) - .map_err(|err| RpcError::Internal(anyhow!(err.to_string())))?; + .context("Failed to subscribe compaction event")?; let stream = self .inner @@ -1430,7 +1441,10 @@ impl HummockMetaClient for MetaClient { #[async_trait] impl TelemetryInfoFetcher for MetaClient { async fn fetch_telemetry_info(&self) -> std::result::Result, String> { - let resp = self.get_telemetry_info().await.map_err(|e| e.to_string())?; + let resp = self + .get_telemetry_info() + .await + .map_err(|e| e.to_report_string())?; let tracking_id = resp.get_tracking_id().ok(); Ok(tracking_id.map(|id| id.to_owned())) } @@ -1566,12 +1580,12 @@ impl MetaMemberManagement { } }; if let Err(err) = client { - tracing::warn!("failed to create client from {}: {}", addr, err); + tracing::warn!(%addr, error = %err.as_report(), "failed to create client"); continue; } match client.unwrap().members(MembersRequest {}).await { Err(err) => { - tracing::warn!("failed to fetch members from {}: {}", addr, err); + tracing::warn!(%addr, error = %err.as_report(), "failed to fetch members"); continue; } Ok(resp) => { @@ -1688,7 +1702,7 @@ impl GrpcMetaClient { let tick_result = member_management.refresh_members().await; if let Err(e) = tick_result.as_ref() { - tracing::warn!("refresh meta member client failed {}", e); + tracing::warn!(error = %e.as_report(), "refresh meta member client failed"); } if let Some(sender) = event { @@ -1769,9 +1783,9 @@ impl GrpcMetaClient { } Err(e) => { tracing::warn!( - "Failed to connect to meta server {}, trying again: {}", + error = %e.as_report(), + "Failed to connect to meta server {}, trying again", addr, - e ) } } @@ -1940,7 +1954,7 @@ impl GrpcMetaClient { .is_ok() { if let Ok(Err(e)) = result_receiver.await { - tracing::warn!("force refresh meta client failed {}", e); + tracing::warn!(error = %e.as_report(), "force refresh meta client failed"); } } else { tracing::debug!("skipping the current refresh, somewhere else is already doing it") diff --git a/src/source/src/common.rs b/src/source/src/common.rs deleted file mode 100644 index aa870967af7f7..0000000000000 --- a/src/source/src/common.rs +++ /dev/null @@ -1,52 +0,0 @@ -// Copyright 2024 RisingWave Labs -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use risingwave_common::array::{ArrayRef, DataChunk}; -use risingwave_common::error::Result; -use risingwave_common::types::Datum; -use risingwave_common::util::iter_util::ZipEqFast; -use risingwave_connector::source::SourceColumnDesc; - -pub(crate) trait SourceChunkBuilder { - fn build_columns<'a>( - column_descs: &[SourceColumnDesc], - rows: impl IntoIterator>, - chunk_size: usize, - ) -> Result> { - let mut builders: Vec<_> = column_descs - .iter() - .map(|k| k.data_type.create_array_builder(chunk_size)) - .collect(); - - for row in rows { - for (datum, builder) in row.iter().zip_eq_fast(&mut builders) { - builder.append(datum); - } - } - - Ok(builders - .into_iter() - .map(|builder| builder.finish().into()) - .collect()) - } - - fn build_datachunk( - column_desc: &[SourceColumnDesc], - rows: &[Vec], - chunk_size: usize, - ) -> Result { - let columns = Self::build_columns(column_desc, rows, chunk_size)?; - Ok(DataChunk::new(columns, rows.len())) - } -} diff --git a/src/sqlparser/src/ast/ddl.rs b/src/sqlparser/src/ast/ddl.rs index 2c49a5654658c..59f74a3ad3acb 100644 --- a/src/sqlparser/src/ast/ddl.rs +++ b/src/sqlparser/src/ast/ddl.rs @@ -20,6 +20,7 @@ use core::fmt; #[cfg(feature = "serde")] use serde::{Deserialize, Serialize}; +use super::ConnectorSchema; use crate::ast::{ display_comma_separated, display_separated, DataType, Expr, Ident, ObjectName, SetVariableValue, }; @@ -84,8 +85,11 @@ pub enum AlterTableOperation { ChangeOwner { new_owner_name: Ident }, /// `SET SCHEMA ` SetSchema { new_schema_name: ObjectName }, - /// `SET PARALLELISM TO ` - SetParallelism { parallelism: SetVariableValue }, + /// `SET PARALLELISM TO [ DEFERRED ]` + SetParallelism { + parallelism: SetVariableValue, + deferred: bool, + }, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -95,9 +99,10 @@ pub enum AlterIndexOperation { RenameIndex { index_name: ObjectName, }, - /// `SET PARALLELISM TO ` + /// `SET PARALLELISM TO [ DEFERRED ]` SetParallelism { parallelism: SetVariableValue, + deferred: bool, }, } @@ -114,9 +119,10 @@ pub enum AlterViewOperation { SetSchema { new_schema_name: ObjectName, }, - /// `SET PARALLELISM TO ` + /// `SET PARALLELISM TO [ DEFERRED ]` SetParallelism { parallelism: SetVariableValue, + deferred: bool, }, } @@ -133,9 +139,10 @@ pub enum AlterSinkOperation { SetSchema { new_schema_name: ObjectName, }, - /// `SET PARALLELISM TO ` + /// `SET PARALLELISM TO [ DEFERRED ]` SetParallelism { parallelism: SetVariableValue, + deferred: bool, }, } @@ -147,6 +154,7 @@ pub enum AlterSourceOperation { AddColumn { column_def: ColumnDef }, ChangeOwner { new_owner_name: Ident }, SetSchema { new_schema_name: ObjectName }, + FormatEncode { connector_schema: ConnectorSchema }, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -244,8 +252,16 @@ impl fmt::Display for AlterTableOperation { AlterTableOperation::SetSchema { new_schema_name } => { write!(f, "SET SCHEMA {}", new_schema_name) } - AlterTableOperation::SetParallelism { parallelism } => { - write!(f, "SET PARALLELISM TO {}", parallelism) + AlterTableOperation::SetParallelism { + parallelism, + deferred, + } => { + write!( + f, + "SET PARALLELISM TO {} {}", + parallelism, + if *deferred { " DEFERRED" } else { "" } + ) } } } @@ -257,8 +273,16 @@ impl fmt::Display for AlterIndexOperation { AlterIndexOperation::RenameIndex { index_name } => { write!(f, "RENAME TO {index_name}") } - AlterIndexOperation::SetParallelism { parallelism } => { - write!(f, "SET PARALLELISM TO {}", parallelism) + AlterIndexOperation::SetParallelism { + parallelism, + deferred, + } => { + write!( + f, + "SET PARALLELISM TO {} {}", + parallelism, + if *deferred { " DEFERRED" } else { "" } + ) } } } @@ -276,8 +300,16 @@ impl fmt::Display for AlterViewOperation { AlterViewOperation::SetSchema { new_schema_name } => { write!(f, "SET SCHEMA {}", new_schema_name) } - AlterViewOperation::SetParallelism { parallelism } => { - write!(f, "SET PARALLELISM TO {}", parallelism) + AlterViewOperation::SetParallelism { + parallelism, + deferred, + } => { + write!( + f, + "SET PARALLELISM TO {} {}", + parallelism, + if *deferred { " DEFERRED" } else { "" } + ) } } } @@ -295,8 +327,16 @@ impl fmt::Display for AlterSinkOperation { AlterSinkOperation::SetSchema { new_schema_name } => { write!(f, "SET SCHEMA {}", new_schema_name) } - AlterSinkOperation::SetParallelism { parallelism } => { - write!(f, "SET PARALLELISM TO {}", parallelism) + AlterSinkOperation::SetParallelism { + parallelism, + deferred, + } => { + write!( + f, + "SET PARALLELISM TO {} {}", + parallelism, + if *deferred { " DEFERRED" } else { "" } + ) } } } @@ -317,6 +357,9 @@ impl fmt::Display for AlterSourceOperation { AlterSourceOperation::SetSchema { new_schema_name } => { write!(f, "SET SCHEMA {}", new_schema_name) } + AlterSourceOperation::FormatEncode { connector_schema } => { + write!(f, "{connector_schema}") + } } } } diff --git a/src/sqlparser/src/ast/mod.rs b/src/sqlparser/src/ast/mod.rs index eef14722ee841..7051c10862d44 100644 --- a/src/sqlparser/src/ast/mod.rs +++ b/src/sqlparser/src/ast/mod.rs @@ -26,6 +26,7 @@ use alloc::{ vec::Vec, }; use core::fmt; +use core::fmt::Display; use itertools::Itertools; #[cfg(feature = "serde")] @@ -55,7 +56,7 @@ pub use crate::ast::ddl::{ AlterIndexOperation, AlterSinkOperation, AlterSourceOperation, AlterViewOperation, }; use crate::keywords::Keyword; -use crate::parser::{Parser, ParserError}; +use crate::parser::{IncludeOption, IncludeOptionItem, Parser, ParserError}; pub struct DisplaySeparated<'a, T> where @@ -814,10 +815,10 @@ impl fmt::Display for WindowFrameUnits { pub enum WindowFrameBound { /// `CURRENT ROW` CurrentRow, - /// ` PRECEDING` or `UNBOUNDED PRECEDING` - Preceding(Option), - /// ` FOLLOWING` or `UNBOUNDED FOLLOWING`. - Following(Option), + /// ` PRECEDING` or `UNBOUNDED PRECEDING` + Preceding(Option>), + /// ` FOLLOWING` or `UNBOUNDED FOLLOWING`. + Following(Option>), } impl fmt::Display for WindowFrameBound { @@ -1117,6 +1118,8 @@ pub enum Statement { name: ObjectName, /// Optional schema columns: Vec, + // The wildchar position in columns defined in sql. Only exist when using external schema. + wildcard_idx: Option, constraints: Vec, with_options: Vec, /// Optional schema of the external source with which the table is created @@ -1130,7 +1133,7 @@ pub enum Statement { /// `FROM cdc_source TABLE database_name.table_name` cdc_table_info: Option, /// `INCLUDE a AS b INCLUDE c` - include_column_options: Vec<(Ident, Option)>, + include_column_options: IncludeOption, }, /// CREATE INDEX CreateIndex { @@ -1166,6 +1169,7 @@ pub enum Statement { returns: Option, /// Optional parameters. params: CreateFunctionBody, + with_options: CreateFunctionWithOptions, }, /// CREATE AGGREGATE /// @@ -1534,6 +1538,7 @@ impl fmt::Display for Statement { args, returns, params, + with_options, } => { write!( f, @@ -1548,6 +1553,7 @@ impl fmt::Display for Statement { write!(f, " {}", return_type)?; } write!(f, "{params}")?; + write!(f, "{with_options}")?; Ok(()) } Statement::CreateAggregate { @@ -1606,6 +1612,7 @@ impl fmt::Display for Statement { Statement::CreateTable { name, columns, + wildcard_idx, constraints, with_options, or_replace, @@ -1634,7 +1641,7 @@ impl fmt::Display for Statement { name = name, )?; if !columns.is_empty() || !constraints.is_empty() { - write!(f, " {}", fmt_create_items(columns, constraints, source_watermarks)?)?; + write!(f, " {}", fmt_create_items(columns, constraints, source_watermarks, *wildcard_idx)?)?; } else if query.is_none() { // PostgreSQL allows `CREATE TABLE t ();`, but requires empty parens write!(f, " ()")?; @@ -1644,12 +1651,20 @@ impl fmt::Display for Statement { } if !include_column_options.is_empty() { // (Ident, Option) write!(f, "{}", display_comma_separated( - include_column_options.iter().map(|(a, b)| { - if let Some(b) = b { - format!("INCLUDE {} AS {}", a, b) - } else { - format!("INCLUDE {}", a) - } + include_column_options.iter().map(|option_item: &IncludeOptionItem| { + format!("INCLUDE {}{}{}", + option_item.column_type, + if let Some(inner_field) = &option_item.inner_field { + format!(" {}", inner_field) + } else { + "".into() + } + , if let Some(alias) = &option_item.column_alias { + format!(" AS {}", alias) + } else { + "".into() + } + ) }).collect_vec().as_slice() ))?; } @@ -2631,6 +2646,24 @@ impl fmt::Display for FunctionDefinition { } } +impl FunctionDefinition { + /// Returns the function definition as a string slice. + pub fn as_str(&self) -> &str { + match self { + FunctionDefinition::SingleQuotedDef(s) => s, + FunctionDefinition::DoubleDollarDef(s) => s, + } + } + + /// Returns the function definition as a string. + pub fn into_string(self) -> String { + match self { + FunctionDefinition::SingleQuotedDef(s) => s, + FunctionDefinition::DoubleDollarDef(s) => s, + } + } +} + /// Return types of a function. #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] @@ -2707,6 +2740,57 @@ impl fmt::Display for CreateFunctionBody { Ok(()) } } +#[derive(Debug, Default, Clone, PartialEq, Eq, Hash)] +#[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] +pub struct CreateFunctionWithOptions { + /// Always retry on network errors. + pub always_retry_on_network_error: Option, +} + +/// TODO(kwannoel): Generate from the struct definition instead. +impl CreateFunctionWithOptions { + fn is_empty(&self) -> bool { + self.always_retry_on_network_error.is_none() + } +} + +/// TODO(kwannoel): Generate from the struct definition instead. +impl TryFrom> for CreateFunctionWithOptions { + type Error = ParserError; + + fn try_from(with_options: Vec) -> Result { + let mut always_retry_on_network_error = None; + for option in with_options { + if option.name.to_string().to_lowercase() == "always_retry_on_network_error" { + always_retry_on_network_error = Some(option.value == Value::Boolean(true)); + } else { + return Err(ParserError::ParserError(format!( + "Unsupported option: {}", + option.name + ))); + } + } + Ok(Self { + always_retry_on_network_error, + }) + } +} + +impl Display for CreateFunctionWithOptions { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + if self.is_empty() { + return Ok(()); + } + let mut options = vec![]; + if let Some(always_retry_on_network_error) = self.always_retry_on_network_error { + options.push(format!( + "ALWAYS_RETRY_NETWORK_ERRORS = {}", + always_retry_on_network_error + )); + } + write!(f, " WITH ( {} )", display_comma_separated(&options)) + } +} #[derive(Debug, Clone, PartialEq, PartialOrd, Eq, Ord, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] @@ -2907,4 +2991,50 @@ mod tests { }; assert_eq!("NOT true IS NOT FALSE", format!("{}", unary_op)); } + + #[test] + fn test_create_function_display() { + let create_function = Statement::CreateFunction { + temporary: false, + or_replace: false, + name: ObjectName(vec![Ident::new_unchecked("foo")]), + args: Some(vec![OperateFunctionArg::unnamed(DataType::Int)]), + returns: Some(CreateFunctionReturns::Value(DataType::Int)), + params: CreateFunctionBody { + language: Some(Ident::new_unchecked("python")), + behavior: Some(FunctionBehavior::Immutable), + as_: Some(FunctionDefinition::SingleQuotedDef("SELECT 1".to_string())), + return_: None, + using: None, + }, + with_options: CreateFunctionWithOptions { + always_retry_on_network_error: None, + }, + }; + assert_eq!( + "CREATE FUNCTION foo(INT) RETURNS INT LANGUAGE python IMMUTABLE AS 'SELECT 1'", + format!("{}", create_function) + ); + let create_function = Statement::CreateFunction { + temporary: false, + or_replace: false, + name: ObjectName(vec![Ident::new_unchecked("foo")]), + args: Some(vec![OperateFunctionArg::unnamed(DataType::Int)]), + returns: Some(CreateFunctionReturns::Value(DataType::Int)), + params: CreateFunctionBody { + language: Some(Ident::new_unchecked("python")), + behavior: Some(FunctionBehavior::Immutable), + as_: Some(FunctionDefinition::SingleQuotedDef("SELECT 1".to_string())), + return_: None, + using: None, + }, + with_options: CreateFunctionWithOptions { + always_retry_on_network_error: Some(true), + }, + }; + assert_eq!( + "CREATE FUNCTION foo(INT) RETURNS INT LANGUAGE python IMMUTABLE AS 'SELECT 1' WITH ( ALWAYS_RETRY_NETWORK_ERRORS = true )", + format!("{}", create_function) + ); + } } diff --git a/src/sqlparser/src/ast/statement.rs b/src/sqlparser/src/ast/statement.rs index 5624a74c621ed..1b73edc1150da 100644 --- a/src/sqlparser/src/ast/statement.rs +++ b/src/sqlparser/src/ast/statement.rs @@ -26,7 +26,7 @@ use crate::ast::{ display_comma_separated, display_separated, ColumnDef, ObjectName, SqlOption, TableConstraint, }; use crate::keywords::Keyword; -use crate::parser::{IsOptional, Parser, ParserError, UPSTREAM_SOURCE_KEY}; +use crate::parser::{IncludeOption, IsOptional, Parser, ParserError, UPSTREAM_SOURCE_KEY}; use crate::tokenizer::Token; /// Consumes token from the parser into an AST node. @@ -80,18 +80,21 @@ macro_rules! impl_fmt_display { pub struct CreateSourceStatement { pub if_not_exists: bool, pub columns: Vec, + // The wildchar position in columns defined in sql. Only exist when using external schema. + pub wildcard_idx: Option, pub constraints: Vec, pub source_name: ObjectName, pub with_properties: WithProperties, pub source_schema: CompatibleSourceSchema, pub source_watermarks: Vec, - pub include_column_options: Vec<(Ident, Option)>, + pub include_column_options: IncludeOption, } #[derive(Debug, Clone, PartialEq, Eq, Hash)] #[cfg_attr(feature = "serde", derive(Serialize, Deserialize))] pub enum Format { Native, + None, // Keyword::NONE Debezium, // Keyword::DEBEZIUM DebeziumMongo, // Keyword::DEBEZIUM_MONGO Maxwell, // Keyword::MAXWELL @@ -114,6 +117,7 @@ impl fmt::Display for Format { Format::Canal => "CANAL", Format::Upsert => "UPSERT", Format::Plain => "PLAIN", + Format::None => "NONE", } ) } @@ -129,9 +133,10 @@ impl Format { "PLAIN" => Format::Plain, "UPSERT" => Format::Upsert, "NATIVE" => Format::Native, // used internally for schema change + "NONE" => Format::None, // used by iceberg _ => { return Err(ParserError::ParserError( - "expected CANAL | PROTOBUF | DEBEZIUM | MAXWELL | PLAIN | NATIVE after FORMAT" + "expected CANAL | PROTOBUF | DEBEZIUM | MAXWELL | PLAIN | NATIVE | NONE after FORMAT" .to_string(), )) } @@ -147,6 +152,7 @@ pub enum Encode { Protobuf, // Keyword::PROTOBUF Json, // Keyword::JSON Bytes, // Keyword::BYTES + None, // Keyword::None Native, Template, } @@ -165,6 +171,7 @@ impl fmt::Display for Encode { Encode::Bytes => "BYTES", Encode::Native => "NATIVE", Encode::Template => "TEMPLATE", + Encode::None => "NONE", } ) } @@ -180,8 +187,9 @@ impl Encode { "JSON" => Encode::Json, "TEMPLATE" => Encode::Template, "NATIVE" => Encode::Native, // used internally for schema change + "NONE" => Encode::None, // used by iceberg _ => return Err(ParserError::ParserError( - "expected AVRO | BYTES | CSV | PROTOBUF | JSON | NATIVE | TEMPLATE after Encode" + "expected AVRO | BYTES | CSV | PROTOBUF | JSON | NATIVE | TEMPLATE | NONE after Encode" .to_string(), )), }) @@ -213,12 +221,16 @@ impl Parser { // row format for cdc source must be debezium json // row format for nexmark source must be native // default row format for datagen source is native + // FIXME: parse input `connector` to enum type instead using string here if connector.contains("-cdc") { let expected = if cdc_source_job { ConnectorSchema::plain_json() + } else if connector.contains("mongodb") { + ConnectorSchema::debezium_mongo_json() } else { ConnectorSchema::debezium_json() }; + if self.peek_source_schema_format() { let schema = parse_source_schema(self)?.into_v2(); if schema != expected { @@ -247,6 +259,18 @@ impl Parser { } else { ConnectorSchema::native().into() }) + } else if connector.contains("iceberg") { + let expected = ConnectorSchema::none(); + if self.peek_source_schema_format() { + let schema = parse_source_schema(self)?.into_v2(); + if schema != expected { + return Err(ParserError::ParserError(format!( + "Row format for iceberg connectors should be \ + either omitted or set to `{expected}`", + ))); + } + } + Ok(expected.into()) } else { Ok(parse_source_schema(self)?) } @@ -293,6 +317,14 @@ impl ConnectorSchema { } } + pub const fn debezium_mongo_json() -> Self { + ConnectorSchema { + format: Format::DebeziumMongo, + row_encode: Encode::Json, + row_options: Vec::new(), + } + } + /// Create a new source schema with `Native` format and encoding. pub const fn native() -> Self { ConnectorSchema { @@ -302,6 +334,16 @@ impl ConnectorSchema { } } + /// Create a new source schema with `None` format and encoding. + /// Used for self-explanatory source like iceberg. + pub const fn none() -> Self { + ConnectorSchema { + format: Format::None, + row_encode: Encode::None, + row_options: Vec::new(), + } + } + pub fn row_options(&self) -> &[SqlOption] { self.row_options.as_ref() } @@ -325,7 +367,8 @@ impl ParseTo for CreateSourceStatement { impl_parse_to!(source_name: ObjectName, p); // parse columns - let (columns, constraints, source_watermarks) = p.parse_columns_with_watermark()?; + let (columns, constraints, source_watermarks, wildcard_idx) = + p.parse_columns_with_watermark()?; let include_options = p.parse_include_options()?; let with_options = p.parse_with_properties()?; @@ -343,6 +386,7 @@ impl ParseTo for CreateSourceStatement { Ok(Self { if_not_exists, columns, + wildcard_idx, constraints, source_name, with_properties: WithProperties(with_options), @@ -357,11 +401,28 @@ pub(super) fn fmt_create_items( columns: &[ColumnDef], constraints: &[TableConstraint], watermarks: &[SourceWatermark], + wildcard_idx: Option, ) -> std::result::Result { let mut items = String::new(); - let has_items = !columns.is_empty() || !constraints.is_empty() || !watermarks.is_empty(); + let has_items = !columns.is_empty() + || !constraints.is_empty() + || !watermarks.is_empty() + || wildcard_idx.is_some(); has_items.then(|| write!(&mut items, "(")); - write!(&mut items, "{}", display_comma_separated(columns))?; + if let Some(wildcard_idx) = wildcard_idx { + let (columns_l, columns_r) = columns.split_at(wildcard_idx); + write!(&mut items, "{}", display_comma_separated(columns_l))?; + if !columns_l.is_empty() { + write!(&mut items, ", ")?; + } + write!(&mut items, "{}", Token::Mul)?; + if !columns_r.is_empty() { + write!(&mut items, ", ")?; + } + write!(&mut items, "{}", display_comma_separated(columns_r))?; + } else { + write!(&mut items, "{}", display_comma_separated(columns))?; + } if !columns.is_empty() && (!constraints.is_empty() || !watermarks.is_empty()) { write!(&mut items, ", ")?; } @@ -380,7 +441,12 @@ impl fmt::Display for CreateSourceStatement { impl_fmt_display!(if_not_exists => [Keyword::IF, Keyword::NOT, Keyword::EXISTS], v, self); impl_fmt_display!(source_name, v, self); - let items = fmt_create_items(&self.columns, &self.constraints, &self.source_watermarks)?; + let items = fmt_create_items( + &self.columns, + &self.constraints, + &self.source_watermarks, + self.wildcard_idx, + )?; if !items.is_empty() { v.push(items); } @@ -689,6 +755,7 @@ pub enum UserOption { NoLogin, EncryptedPassword(AstString), Password(Option), + OAuth(Vec), } impl fmt::Display for UserOption { @@ -705,6 +772,9 @@ impl fmt::Display for UserOption { UserOption::EncryptedPassword(p) => write!(f, "ENCRYPTED PASSWORD {}", p), UserOption::Password(None) => write!(f, "PASSWORD NULL"), UserOption::Password(Some(p)) => write!(f, "PASSWORD {}", p), + UserOption::OAuth(options) => { + write!(f, "({})", display_comma_separated(options.as_slice())) + } } } } @@ -792,10 +862,14 @@ impl ParseTo for UserOptions { UserOption::EncryptedPassword(AstString::parse_to(parser)?), ) } + Keyword::OAUTH => { + let options = parser.parse_options()?; + (&mut builder.password, UserOption::OAuth(options)) + } _ => { parser.expected( "SUPERUSER | NOSUPERUSER | CREATEDB | NOCREATEDB | LOGIN \ - | NOLOGIN | CREATEUSER | NOCREATEUSER | [ENCRYPTED] PASSWORD | NULL", + | NOLOGIN | CREATEUSER | NOCREATEUSER | [ENCRYPTED] PASSWORD | NULL | OAUTH", token, )?; unreachable!() @@ -805,7 +879,7 @@ impl ParseTo for UserOptions { } else { parser.expected( "SUPERUSER | NOSUPERUSER | CREATEDB | NOCREATEDB | LOGIN | NOLOGIN \ - | CREATEUSER | NOCREATEUSER | [ENCRYPTED] PASSWORD | NULL", + | CREATEUSER | NOCREATEUSER | [ENCRYPTED] PASSWORD | NULL | OAUTH", token, )? } diff --git a/src/sqlparser/src/keywords.rs b/src/sqlparser/src/keywords.rs index dae6529376c4c..a3cc9013a21ef 100644 --- a/src/sqlparser/src/keywords.rs +++ b/src/sqlparser/src/keywords.rs @@ -70,6 +70,7 @@ define_keywords!( ABORT, ABS, ACTION, + ADAPTIVE, ADD, AGGREGATE, ALL, @@ -183,6 +184,7 @@ define_keywords!( DECLARE, DEFAULT, DEFERRABLE, + DEFERRED, DELETE, DELIMITED, DENSE_RANK, @@ -342,6 +344,7 @@ define_keywords!( NULLIF, NULLS, NUMERIC, + OAUTH, OBJECT, OCCURRENCES_REGEX, OCTET_LENGTH, diff --git a/src/sqlparser/src/parser.rs b/src/sqlparser/src/parser.rs index 5bfda592f3b27..22f0350024141 100644 --- a/src/sqlparser/src/parser.rs +++ b/src/sqlparser/src/parser.rs @@ -80,6 +80,17 @@ pub enum IsLateral { use IsLateral::*; +pub type IncludeOption = Vec; + +#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] +#[derive(Eq, Clone, Debug, PartialEq, Hash)] +pub struct IncludeOptionItem { + pub column_type: Ident, + pub column_alias: Option, + pub inner_field: Option, + pub header_inner_expect_type: Option, +} + #[derive(Debug)] pub enum WildcardOrExpr { Expr(Expr), @@ -131,7 +142,12 @@ impl fmt::Display for ParserError { #[cfg(feature = "std")] impl std::error::Error for ParserError {} -type ColumnsDefTuple = (Vec, Vec, Vec); +type ColumnsDefTuple = ( + Vec, + Vec, + Vec, + Option, +); /// Reference: /// @@ -907,7 +923,7 @@ impl Parser { }) } - /// Parse `CURRENT ROW` or `{ | UNBOUNDED } { PRECEDING | FOLLOWING }` + /// Parse `CURRENT ROW` or `{ | UNBOUNDED } { PRECEDING | FOLLOWING }` pub fn parse_window_frame_bound(&mut self) -> Result { if self.parse_keywords(&[Keyword::CURRENT, Keyword::ROW]) { Ok(WindowFrameBound::CurrentRow) @@ -915,7 +931,7 @@ impl Parser { let rows = if self.parse_keyword(Keyword::UNBOUNDED) { None } else { - Some(self.parse_literal_uint()?) + Some(Box::new(self.parse_expr()?)) }; if self.parse_keyword(Keyword::PRECEDING) { Ok(WindowFrameBound::Preceding(rows)) @@ -2222,7 +2238,8 @@ impl Parser { }; let params = self.parse_create_function_body()?; - + let with_options = self.parse_options_with_preceding_keyword(Keyword::WITH)?; + let with_options = with_options.try_into()?; Ok(Statement::CreateFunction { or_replace, temporary, @@ -2230,6 +2247,7 @@ impl Parser { args, returns: return_type, params, + with_options, }) } @@ -2361,7 +2379,7 @@ impl Parser { // | CREATEDB | NOCREATEDB // | CREATEUSER | NOCREATEUSER // | LOGIN | NOLOGIN - // | [ ENCRYPTED ] PASSWORD 'password' | PASSWORD NULL + // | [ ENCRYPTED ] PASSWORD 'password' | PASSWORD NULL | OAUTH fn parse_create_user(&mut self) -> Result { Ok(Statement::CreateUser(CreateUserStatement::parse_to(self)?)) } @@ -2455,7 +2473,8 @@ impl Parser { let if_not_exists = self.parse_keywords(&[Keyword::IF, Keyword::NOT, Keyword::EXISTS]); let table_name = self.parse_object_name()?; // parse optional column list (schema) and watermarks on source. - let (columns, constraints, source_watermarks) = self.parse_columns_with_watermark()?; + let (columns, constraints, source_watermarks, wildcard_idx) = + self.parse_columns_with_watermark()?; let append_only = if self.parse_keyword(Keyword::APPEND) { self.expect_keyword(Keyword::ONLY)?; @@ -2507,6 +2526,7 @@ impl Parser { name: table_name, temporary, columns, + wildcard_idx, constraints, with_options, or_replace, @@ -2520,16 +2540,46 @@ impl Parser { }) } - pub fn parse_include_options(&mut self) -> Result)>, ParserError> { + pub fn parse_include_options(&mut self) -> Result { let mut options = vec![]; while self.parse_keyword(Keyword::INCLUDE) { - let add_column = self.parse_identifier()?; + let column_type = self.parse_identifier()?; + + let mut column_inner_field = None; + let mut header_inner_expect_type = None; + if let Token::SingleQuotedString(inner_field) = self.peek_token().token { + self.next_token(); + column_inner_field = Some(inner_field); + + if let Token::Word(w) = self.peek_token().token { + match w.keyword { + Keyword::BYTEA => { + header_inner_expect_type = Some(DataType::Bytea); + self.next_token(); + } + Keyword::VARCHAR => { + header_inner_expect_type = Some(DataType::Varchar); + self.next_token(); + } + _ => { + // default to bytea + header_inner_expect_type = Some(DataType::Bytea); + } + } + } + } + + let mut column_alias = None; if self.parse_keyword(Keyword::AS) { - let column_alias = self.parse_identifier()?; - options.push((add_column, Some(column_alias))); - } else { - options.push((add_column, None)); + column_alias = Some(self.parse_identifier()?); } + + options.push(IncludeOptionItem { + column_type, + inner_field: column_inner_field, + column_alias, + header_inner_expect_type, + }); } Ok(options) } @@ -2538,12 +2588,21 @@ impl Parser { let mut columns = vec![]; let mut constraints = vec![]; let mut watermarks = vec![]; + let mut wildcard_idx = None; if !self.consume_token(&Token::LParen) || self.consume_token(&Token::RParen) { - return Ok((columns, constraints, watermarks)); + return Ok((columns, constraints, watermarks, wildcard_idx)); } loop { - if let Some(constraint) = self.parse_optional_table_constraint()? { + if self.consume_token(&Token::Mul) { + if wildcard_idx.is_none() { + wildcard_idx = Some(columns.len()); + } else { + return Err(ParserError::ParserError( + "At most 1 wildcard is allowed in source definetion".to_string(), + )); + } + } else if let Some(constraint) = self.parse_optional_table_constraint()? { constraints.push(constraint); } else if let Some(watermark) = self.parse_optional_watermark()? { watermarks.push(watermark); @@ -2567,7 +2626,7 @@ impl Parser { } } - Ok((columns, constraints, watermarks)) + Ok((columns, constraints, watermarks, wildcard_idx)) } fn parse_column_def(&mut self) -> Result { @@ -2960,7 +3019,12 @@ impl Parser { let value = self.parse_set_variable()?; - AlterTableOperation::SetParallelism { parallelism: value } + let deferred = self.parse_keyword(Keyword::DEFERRED); + + AlterTableOperation::SetParallelism { + parallelism: value, + deferred, + } } else { return self.expected("SCHEMA/PARALLELISM after SET", self.peek_token()); } @@ -3039,7 +3103,12 @@ impl Parser { let value = self.parse_set_variable()?; - AlterIndexOperation::SetParallelism { parallelism: value } + let deferred = self.parse_keyword(Keyword::DEFERRED); + + AlterIndexOperation::SetParallelism { + parallelism: value, + deferred, + } } else { return self.expected("PARALLELISM after SET", self.peek_token()); } @@ -3085,7 +3154,12 @@ impl Parser { let value = self.parse_set_variable()?; - AlterViewOperation::SetParallelism { parallelism: value } + let deferred = self.parse_keyword(Keyword::DEFERRED); + + AlterViewOperation::SetParallelism { + parallelism: value, + deferred, + } } else { return self.expected("SCHEMA/PARALLELISM after SET", self.peek_token()); } @@ -3137,8 +3211,12 @@ impl Parser { } let value = self.parse_set_variable()?; + let deferred = self.parse_keyword(Keyword::DEFERRED); - AlterSinkOperation::SetParallelism { parallelism: value } + AlterSinkOperation::SetParallelism { + parallelism: value, + deferred, + } } else { return self.expected("SCHEMA/PARALLELISM after SET", self.peek_token()); } @@ -3183,6 +3261,9 @@ impl Parser { } else { return self.expected("SCHEMA after SET", self.peek_token()); } + } else if self.peek_nth_any_of_keywords(0, &[Keyword::FORMAT]) { + let connector_schema = self.parse_schema()?.unwrap(); + AlterSourceOperation::FormatEncode { connector_schema } } else { return self.expected( "RENAME, ADD COLUMN or OWNER TO or SET after ALTER SOURCE", diff --git a/src/sqlparser/tests/sqlparser_postgres.rs b/src/sqlparser/tests/sqlparser_postgres.rs index 99e2c185fdcff..6a5dec5d809c1 100644 --- a/src/sqlparser/tests/sqlparser_postgres.rs +++ b/src/sqlparser/tests/sqlparser_postgres.rs @@ -765,6 +765,7 @@ fn parse_create_function() { )), ..Default::default() }, + with_options: Default::default(), } ); @@ -786,7 +787,8 @@ fn parse_create_function() { "select $1 - $2;".into() )), ..Default::default() - } + }, + with_options: Default::default(), }, ); @@ -811,7 +813,8 @@ fn parse_create_function() { right: Box::new(Expr::Parameter { index: 2 }), }), ..Default::default() - } + }, + with_options: Default::default(), }, ); @@ -842,6 +845,7 @@ fn parse_create_function() { }), ..Default::default() }, + with_options: Default::default(), } ); @@ -865,6 +869,7 @@ fn parse_create_function() { return_: Some(Expr::Identifier("a".into())), ..Default::default() }, + with_options: Default::default(), } ); } diff --git a/src/sqlparser/tests/testdata/create.yaml b/src/sqlparser/tests/testdata/create.yaml index 69167205591a2..94873541e1d81 100644 --- a/src/sqlparser/tests/testdata/create.yaml +++ b/src/sqlparser/tests/testdata/create.yaml @@ -35,13 +35,13 @@ Near "pad CHARACTER VARYING) FROM sbtest" - input: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://') formatted_sql: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.location = 'file://') - formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "location", quote_style: None }]), value: SingleQuotedString("file://") }] }), source_watermarks: [], include_column_options: [] } }' + formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "location", quote_style: None }]), value: SingleQuotedString("file://") }] }), source_watermarks: [], include_column_options: [] } }' - input: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://') formatted_sql: CREATE SOURCE IF NOT EXISTS src WITH (kafka.topic = 'abc', kafka.servers = 'localhost:1001') FORMAT PLAIN ENCODE PROTOBUF (message = 'Foo', schema.registry = 'http://') - formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }] }), source_watermarks: [], include_column_options: [] } }' + formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: true, columns: [], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "src", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "topic", quote_style: None }]), value: SingleQuotedString("abc") }, SqlOption { name: ObjectName([Ident { value: "kafka", quote_style: None }, Ident { value: "servers", quote_style: None }]), value: SingleQuotedString("localhost:1001") }]), source_schema: V2(ConnectorSchema { format: Plain, row_encode: Protobuf, row_options: [SqlOption { name: ObjectName([Ident { value: "message", quote_style: None }]), value: SingleQuotedString("Foo") }, SqlOption { name: ObjectName([Ident { value: "schema", quote_style: None }, Ident { value: "registry", quote_style: None }]), value: SingleQuotedString("http://") }] }), source_watermarks: [], include_column_options: [] } }' - input: CREATE SOURCE bid (auction INTEGER, bidder INTEGER, price INTEGER, WATERMARK FOR auction AS auction - 1, "date_time" TIMESTAMP) with (connector = 'nexmark', nexmark.table.type = 'Bid', nexmark.split.num = '12', nexmark.min.event.gap.in.ns = '0') formatted_sql: CREATE SOURCE bid (auction INT, bidder INT, price INT, "date_time" TIMESTAMP, WATERMARK FOR auction AS auction - 1) WITH (connector = 'nexmark', nexmark.table.type = 'Bid', nexmark.split.num = '12', nexmark.min.event.gap.in.ns = '0') FORMAT NATIVE ENCODE NATIVE - formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: false, columns: [ColumnDef { name: Ident { value: "auction", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "bidder", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "price", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "date_time", quote_style: Some(''"'') }, data_type: Some(Timestamp(false)), collation: None, options: [] }], constraints: [], source_name: ObjectName([Ident { value: "bid", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "connector", quote_style: None }]), value: SingleQuotedString("nexmark") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "table", quote_style: None }, Ident { value: "type", quote_style: None }]), value: SingleQuotedString("Bid") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "split", quote_style: None }, Ident { value: "num", quote_style: None }]), value: SingleQuotedString("12") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "min", quote_style: None }, Ident { value: "event", quote_style: None }, Ident { value: "gap", quote_style: None }, Ident { value: "in", quote_style: None }, Ident { value: "ns", quote_style: None }]), value: SingleQuotedString("0") }]), source_schema: V2(ConnectorSchema { format: Native, row_encode: Native, row_options: [] }), source_watermarks: [SourceWatermark { column: Ident { value: "auction", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "auction", quote_style: None }), op: Minus, right: Value(Number("1")) } }], include_column_options: [] } }' + formatted_ast: 'CreateSource { stmt: CreateSourceStatement { if_not_exists: false, columns: [ColumnDef { name: Ident { value: "auction", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "bidder", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "price", quote_style: None }, data_type: Some(Int), collation: None, options: [] }, ColumnDef { name: Ident { value: "date_time", quote_style: Some(''"'') }, data_type: Some(Timestamp(false)), collation: None, options: [] }], wildcard_idx: None, constraints: [], source_name: ObjectName([Ident { value: "bid", quote_style: None }]), with_properties: WithProperties([SqlOption { name: ObjectName([Ident { value: "connector", quote_style: None }]), value: SingleQuotedString("nexmark") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "table", quote_style: None }, Ident { value: "type", quote_style: None }]), value: SingleQuotedString("Bid") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "split", quote_style: None }, Ident { value: "num", quote_style: None }]), value: SingleQuotedString("12") }, SqlOption { name: ObjectName([Ident { value: "nexmark", quote_style: None }, Ident { value: "min", quote_style: None }, Ident { value: "event", quote_style: None }, Ident { value: "gap", quote_style: None }, Ident { value: "in", quote_style: None }, Ident { value: "ns", quote_style: None }]), value: SingleQuotedString("0") }]), source_schema: V2(ConnectorSchema { format: Native, row_encode: Native, row_options: [] }), source_watermarks: [SourceWatermark { column: Ident { value: "auction", quote_style: None }, expr: BinaryOp { left: Identifier(Ident { value: "auction", quote_style: None }), op: Minus, right: Value(Number("1")) } }], include_column_options: [] } }' - input: CREATE TABLE T (v1 INT, v2 STRUCT) formatted_sql: CREATE TABLE T (v1 INT, v2 STRUCT) - input: CREATE TABLE T (v1 INT, v2 STRUCT>) diff --git a/src/storage/Cargo.toml b/src/storage/Cargo.toml index 3bf150973a045..544afb84c163a 100644 --- a/src/storage/Cargo.toml +++ b/src/storage/Cargo.toml @@ -76,7 +76,7 @@ zstd = { version = "0.13", default-features = false } [target.'cfg(target_os = "linux")'.dependencies] procfs = { version = "0.16", default-features = false } libc = "0.2" -nix = { version = "0.27", features = ["fs", "mman"] } +nix = { version = "0.28", features = ["fs", "mman"] } [target.'cfg(target_os = "macos")'.dependencies] darwin-libproc = { git = "https://github.com/risingwavelabs/darwin-libproc.git", rev = "a502be24bd0971463f5bcbfe035a248d8ba503b7" } @@ -87,7 +87,7 @@ mach2 = "0.4" workspace-hack = { path = "../workspace-hack" } [dev-dependencies] -criterion = { workspace = true, features = ["async_futures"] } +criterion = { workspace = true, features = ["async_futures", "async_tokio"] } expect-test = "1" moka = { version = "0.12", features = ["future"] } risingwave_hummock_sdk = { workspace = true } @@ -139,5 +139,9 @@ harness = false name = "bench_row" harness = false +[[bench]] +name = "bench_imm_compact" +harness = false + [lints] workspace = true diff --git a/src/storage/backup/src/meta_snapshot_v1.rs b/src/storage/backup/src/meta_snapshot_v1.rs index 4cc67d87f6561..f6c457572aaa8 100644 --- a/src/storage/backup/src/meta_snapshot_v1.rs +++ b/src/storage/backup/src/meta_snapshot_v1.rs @@ -20,7 +20,7 @@ use itertools::Itertools; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_hummock_sdk::version::HummockVersion; use risingwave_pb::catalog::{ - Connection, Database, Function, Index, Schema, Sink, Source, Table, View, + Connection, Database, Function, Index, Schema, Sink, Source, Subscription, Table, View, }; use risingwave_pb::hummock::{CompactionGroup, HummockVersionStats}; use risingwave_pb::meta::{SystemParams, TableFragments}; @@ -120,6 +120,7 @@ pub struct ClusterMetadata { pub connection: Vec, pub system_param: SystemParams, pub cluster_id: String, + pub subscription: Vec, } impl ClusterMetadata { @@ -144,6 +145,7 @@ impl ClusterMetadata { Self::encode_prost_message_list(&self.connection.iter().collect_vec(), buf); Self::encode_prost_message(&self.system_param, buf); Self::encode_prost_message(&self.cluster_id, buf); + Self::encode_prost_message_list(&self.subscription.iter().collect_vec(), buf); Ok(()) } @@ -171,6 +173,8 @@ impl ClusterMetadata { let connection: Vec = Self::decode_prost_message_list(&mut buf)?; let system_param: SystemParams = Self::decode_prost_message(&mut buf)?; let cluster_id: String = Self::decode_prost_message(&mut buf)?; + let subscription: Vec = + Self::try_decode_prost_message_list(&mut buf).unwrap_or_else(|| Ok(vec![]))?; Ok(Self { default_cf, @@ -190,6 +194,7 @@ impl ClusterMetadata { connection, system_param, cluster_id, + subscription, }) } @@ -228,6 +233,16 @@ impl ClusterMetadata { } Ok(result) } + + fn try_decode_prost_message_list(buf: &mut &[u8]) -> Option>> + where + T: prost::Message + Default, + { + if buf.is_empty() { + return None; + } + Some(Self::decode_prost_message_list(buf)) + } } #[cfg(test)] diff --git a/src/storage/backup/src/storage.rs b/src/storage/backup/src/storage.rs index f7a1e5e61a35f..54d2eaa741c94 100644 --- a/src/storage/backup/src/storage.rs +++ b/src/storage/backup/src/storage.rs @@ -16,6 +16,7 @@ use std::collections::HashSet; use std::sync::Arc; use itertools::Itertools; +use risingwave_common::config::ObjectStoreConfig; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; use risingwave_object_store::object::{ InMemObjectStore, MonitoredObjectStore, ObjectError, ObjectStoreImpl, ObjectStoreRef, @@ -191,6 +192,7 @@ pub async fn unused() -> ObjectStoreMetaSnapshotStorage { Arc::new(ObjectStoreImpl::InMem(MonitoredObjectStore::new( InMemObjectStore::new(), Arc::new(ObjectStoreMetrics::unused()), + ObjectStoreConfig::default(), ))), ) .await diff --git a/src/storage/benches/bench_compactor.rs b/src/storage/benches/bench_compactor.rs index 1a5f8c94cd110..94ea463eb5af3 100644 --- a/src/storage/benches/bench_compactor.rs +++ b/src/storage/benches/bench_compactor.rs @@ -19,6 +19,7 @@ use criterion::async_executor::FuturesExecutor; use criterion::{criterion_group, criterion_main, Criterion}; use risingwave_common::cache::CachePriority; use risingwave_common::catalog::TableId; +use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_common::hash::VirtualNode; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; @@ -31,8 +32,7 @@ use risingwave_storage::hummock::compactor::{ ConcatSstableIterator, DummyCompactionFilter, TaskConfig, TaskProgress, }; use risingwave_storage::hummock::iterator::{ - ConcatIterator, Forward, ForwardMergeRangeIterator, HummockIterator, - UnorderedMergeIteratorInner, + ConcatIterator, Forward, ForwardMergeRangeIterator, HummockIterator, MergeIterator, }; use risingwave_storage::hummock::multi_builder::{ CapacitySplitTableBuilder, LocalTableBuilderFactory, @@ -42,26 +42,32 @@ use risingwave_storage::hummock::sstable_store::SstableStoreRef; use risingwave_storage::hummock::value::HummockValue; use risingwave_storage::hummock::{ CachePolicy, CompactionDeleteRangeIterator, FileCache, SstableBuilder, SstableBuilderOptions, - SstableIterator, SstableStore, SstableWriterOptions, Xor16FilterBuilder, + SstableIterator, SstableStore, SstableStoreConfig, SstableWriterOptions, Xor16FilterBuilder, +}; +use risingwave_storage::monitor::{ + global_hummock_state_store_metrics, CompactorMetrics, StoreLocalStatistic, }; -use risingwave_storage::monitor::{CompactorMetrics, StoreLocalStatistic}; pub fn mock_sstable_store() -> SstableStoreRef { - let store = InMemObjectStore::new().monitored(Arc::new(ObjectStoreMetrics::unused())); + let store = InMemObjectStore::new().monitored( + Arc::new(ObjectStoreMetrics::unused()), + ObjectStoreConfig::default(), + ); let store = Arc::new(ObjectStoreImpl::InMem(store)); let path = "test".to_string(); - Arc::new(SstableStore::new( + Arc::new(SstableStore::new(SstableStoreConfig { store, path, - 64 << 20, - 128 << 20, - 0, - 64 << 20, - 16, - FileCache::none(), - FileCache::none(), - None, - )) + block_cache_capacity: 64 << 20, + meta_cache_capacity: 128 << 20, + high_priority_ratio: 0, + prefetch_buffer_capacity: 64 << 20, + max_prefetch_block_number: 16, + data_file_cache: FileCache::none(), + meta_file_cache: FileCache::none(), + recent_filter: None, + state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)), + })) } pub fn default_writer_opts() -> SstableWriterOptions { @@ -84,7 +90,8 @@ pub fn test_key_of(idx: usize, epoch: u64) -> FullKey> { ) } -const MAX_KEY_COUNT: usize = 128 * 1024; +/// 8M keys. +const MAX_KEY_COUNT: usize = 8 * 1024 * 1024; async fn build_table( sstable_store: SstableStoreRef, @@ -171,7 +178,7 @@ fn bench_table_scan(c: &mut Criterion) { c.bench_function("bench_table_iterator", |b| { let info1 = info.clone(); - b.to_async(FuturesExecutor) + b.to_async(&runtime) .iter(|| scan_all_table(&info1, sstable_store.clone())); }); } @@ -216,15 +223,29 @@ fn bench_merge_iterator_compactor(c: &mut Criterion) { let sstable_store = mock_sstable_store(); let test_key_size = 256 * 1024; let info1 = runtime - .block_on(async { build_table(sstable_store.clone(), 1, 0..test_key_size, 1).await }); - let info2 = runtime - .block_on(async { build_table(sstable_store.clone(), 2, 0..test_key_size, 1).await }); + .block_on(async { build_table(sstable_store.clone(), 1, 0..test_key_size / 2, 1).await }); + let info2 = runtime.block_on(async { + build_table( + sstable_store.clone(), + 2, + test_key_size / 2..test_key_size, + 1, + ) + .await + }); let level1 = vec![info1, info2]; let info1 = runtime - .block_on(async { build_table(sstable_store.clone(), 3, 0..test_key_size, 2).await }); - let info2 = runtime - .block_on(async { build_table(sstable_store.clone(), 4, 0..test_key_size, 2).await }); + .block_on(async { build_table(sstable_store.clone(), 3, 0..test_key_size / 2, 2).await }); + let info2 = runtime.block_on(async { + build_table( + sstable_store.clone(), + 4, + test_key_size / 2..test_key_size, + 2, + ) + .await + }); let level2 = vec![info1, info2]; let read_options = Arc::new(SstableIteratorReadOptions { cache_policy: CachePolicy::Fill(CachePriority::High), @@ -239,7 +260,7 @@ fn bench_merge_iterator_compactor(c: &mut Criterion) { ConcatIterator::new(level1.clone(), sstable_store.clone(), read_options.clone()), ConcatIterator::new(level2.clone(), sstable_store.clone(), read_options.clone()), ]; - let iter = UnorderedMergeIteratorInner::for_compactor(sub_iters); + let iter = MergeIterator::for_compactor(sub_iters); async move { compact(iter, sstable_store1).await } }); }); @@ -263,7 +284,7 @@ fn bench_merge_iterator_compactor(c: &mut Criterion) { 0, ), ]; - let iter = UnorderedMergeIteratorInner::for_compactor(sub_iters); + let iter = MergeIterator::for_compactor(sub_iters); let sstable_store1 = sstable_store.clone(); async move { compact(iter, sstable_store1).await } }); diff --git a/src/storage/benches/bench_fs_operation.rs b/src/storage/benches/bench_fs_operation.rs index 4f01af01e60cc..0983883fdaa44 100644 --- a/src/storage/benches/bench_fs_operation.rs +++ b/src/storage/benches/bench_fs_operation.rs @@ -94,6 +94,7 @@ fn gen_tokio_files(path: &Path) -> impl IntoIterator impl IntoIterator Vec { + let mut batches = Vec::new(); + for i in 0..batch_count { + let mut batch_data = vec![]; + for j in 0..batch_size { + batch_data.push(( + TableKey(Bytes::copy_from_slice( + format!("test_key_{:08}", j * batch_count + i).as_bytes(), + )), + HummockValue::put(Bytes::copy_from_slice("value".as_bytes())), + )); + } + let batch = SharedBufferBatch::for_test(batch_data, epoch, Default::default()); + batches.push(batch); + } + batches.reverse(); + batches +} + +fn criterion_benchmark(c: &mut Criterion) { + let batches = gen_interleave_shared_buffer_batch(10000, 100, 100); + c.bench_with_input( + BenchmarkId::new("bench-imm-merge", "single-epoch"), + &batches, + |b, batches| { + b.to_async(FuturesExecutor).iter(|| async { + let imm = merge_imms_in_memory(TableId::default(), 0, batches.clone(), None).await; + assert_eq!(imm.key_count(), 10000 * 100); + assert_eq!(imm.value_count(), 10000 * 100); + }) + }, + ); + + let mut later_batches = gen_interleave_shared_buffer_batch(2000, 100, 600); + + for i in 1..5 { + let mut batches = gen_interleave_shared_buffer_batch(2000, 100, 600 - i * 100); + batches.extend(later_batches); + later_batches = batches; + } + + c.bench_with_input( + BenchmarkId::new("bench-imm-merge", "multi-epoch"), + &later_batches, + |b, batches| { + b.to_async(FuturesExecutor).iter(|| async { + let imm = merge_imms_in_memory(TableId::default(), 0, batches.clone(), None).await; + assert_eq!(imm.key_count(), 2000 * 100); + assert_eq!(imm.value_count(), 2000 * 100 * 5); + }) + }, + ); +} + +criterion_group!(benches, criterion_benchmark); +criterion_main!(benches); diff --git a/src/storage/benches/bench_lru_cache.rs b/src/storage/benches/bench_lru_cache.rs index 4eb957084fe2a..43c4193367b83 100644 --- a/src/storage/benches/bench_lru_cache.rs +++ b/src/storage/benches/bench_lru_cache.rs @@ -111,7 +111,7 @@ impl CacheBase for LruCacheImpl { .map(|block| (Arc::new(block), 1)) }) .await?; - Ok(entry.value().clone()) + Ok((*entry).clone()) } } diff --git a/src/storage/benches/bench_merge_iter.rs b/src/storage/benches/bench_merge_iter.rs index 1e44d80279d01..c7b42f1894265 100644 --- a/src/storage/benches/bench_merge_iter.rs +++ b/src/storage/benches/bench_merge_iter.rs @@ -20,8 +20,7 @@ use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion}; use futures::executor::block_on; use risingwave_hummock_sdk::key::TableKey; use risingwave_storage::hummock::iterator::{ - Forward, HummockIterator, HummockIteratorUnion, OrderedMergeIteratorInner, - SkipWatermarkIterator, UnorderedMergeIteratorInner, + Forward, HummockIterator, HummockIteratorUnion, MergeIterator, SkipWatermarkIterator, }; use risingwave_storage::hummock::shared_buffer::shared_buffer_batch::{ SharedBufferBatch, SharedBufferBatchIterator, @@ -97,23 +96,9 @@ fn run_iter>(iter_ref: &RefCell, tota } fn criterion_benchmark(c: &mut Criterion) { - let ordered_merge_iter = RefCell::new(OrderedMergeIteratorInner::new( - gen_interleave_shared_buffer_batch_iter(10000, 100), - )); - - c.bench_with_input( - BenchmarkId::new("bench-merge-iter", "ordered"), - &ordered_merge_iter, - |b, iter_ref| { - b.iter(|| { - run_iter(iter_ref, 100 * 10000); - }); - }, - ); - - let merge_iter = RefCell::new(UnorderedMergeIteratorInner::new( - gen_interleave_shared_buffer_batch_iter(10000, 100), - )); + let merge_iter = RefCell::new(MergeIterator::new(gen_interleave_shared_buffer_batch_iter( + 10000, 100, + ))); c.bench_with_input( BenchmarkId::new("bench-merge-iter", "unordered"), &merge_iter, @@ -125,7 +110,7 @@ fn criterion_benchmark(c: &mut Criterion) { ); let merge_iter = RefCell::new(SkipWatermarkIterator::new( - UnorderedMergeIteratorInner::new(gen_interleave_shared_buffer_batch_iter(10000, 100)), + MergeIterator::new(gen_interleave_shared_buffer_batch_iter(10000, 100)), BTreeMap::new(), )); c.bench_with_input( @@ -138,7 +123,7 @@ fn criterion_benchmark(c: &mut Criterion) { }, ); - let merge_iter = RefCell::new(UnorderedMergeIteratorInner::new( + let merge_iter = RefCell::new(MergeIterator::new( gen_interleave_shared_buffer_batch_enum_iter(10000, 100), )); c.bench_with_input( @@ -150,20 +135,6 @@ fn criterion_benchmark(c: &mut Criterion) { }); }, ); - - let ordered_merge_iter = RefCell::new(OrderedMergeIteratorInner::new( - gen_interleave_shared_buffer_batch_enum_iter(10000, 100), - )); - - c.bench_with_input( - BenchmarkId::new("bench-enum-merge-iter", "ordered"), - &ordered_merge_iter, - |b, iter_ref| { - b.iter(|| { - run_iter(iter_ref, 100 * 10000); - }); - }, - ); } criterion_group!(benches, criterion_benchmark); diff --git a/src/storage/benches/bench_multi_builder.rs b/src/storage/benches/bench_multi_builder.rs index a55cb24fff801..bdc288b4d6925 100644 --- a/src/storage/benches/bench_multi_builder.rs +++ b/src/storage/benches/bench_multi_builder.rs @@ -23,17 +23,17 @@ use criterion::{criterion_group, criterion_main, Criterion}; use futures::future::try_join_all; use itertools::Itertools; use risingwave_common::catalog::TableId; -use risingwave_common::config::ObjectStoreConfig; +use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_hummock_sdk::key::{FullKey, UserKey}; use risingwave_object_store::object::{ObjectStore, ObjectStoreImpl, S3ObjectStore}; use risingwave_storage::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory}; use risingwave_storage::hummock::value::HummockValue; use risingwave_storage::hummock::{ BatchSstableWriterFactory, CachePolicy, FileCache, HummockResult, MemoryLimiter, - SstableBuilder, SstableBuilderOptions, SstableStore, SstableWriterFactory, + SstableBuilder, SstableBuilderOptions, SstableStore, SstableStoreConfig, SstableWriterFactory, SstableWriterOptions, StreamingSstableWriterFactory, Xor16FilterBuilder, }; -use risingwave_storage::monitor::ObjectStoreMetrics; +use risingwave_storage::monitor::{global_hummock_state_store_metrics, ObjectStoreMetrics}; const RANGE: Range = 0..1500000; const VALUE: &[u8] = &[0; 400]; @@ -138,21 +138,22 @@ fn bench_builder( ObjectStoreConfig::default(), ) .await - .monitored(metrics) + .monitored(metrics, ObjectStoreConfig::default()) }); let object_store = Arc::new(ObjectStoreImpl::S3(object_store)); - let sstable_store = Arc::new(SstableStore::new( - object_store, - "test".to_string(), - 64 << 20, - 128 << 20, - 0, - 64 << 20, - 16, - FileCache::none(), - FileCache::none(), - None, - )); + let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig { + store: object_store, + path: "test".to_string(), + block_cache_capacity: 64 << 20, + meta_cache_capacity: 128 << 20, + high_priority_ratio: 0, + prefetch_buffer_capacity: 64 << 20, + max_prefetch_block_number: 16, + data_file_cache: FileCache::none(), + meta_file_cache: FileCache::none(), + recent_filter: None, + state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)), + })); let mut group = c.benchmark_group("bench_multi_builder"); group diff --git a/src/storage/benches/bench_row.rs b/src/storage/benches/bench_row.rs index 6a612e87241cd..49fa52ad5adde 100644 --- a/src/storage/benches/bench_row.rs +++ b/src/storage/benches/bench_row.rs @@ -18,7 +18,6 @@ use std::sync::Arc; use criterion::{criterion_group, criterion_main, Criterion}; use itertools::Itertools; use risingwave_common::catalog::{ColumnDesc, ColumnId}; -use risingwave_common::error::Result; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, Datum, ScalarImpl}; use risingwave_common::util::row_serde::OrderedRowSerde; @@ -104,7 +103,7 @@ fn column_aware_encode(c: &Case) -> Vec> { array } -fn memcmp_decode(c: &Case, bytes: &Vec>) -> Result>> { +fn memcmp_decode(c: &Case, bytes: &Vec>) -> anyhow::Result>> { let serde = OrderedRowSerde::new( c.schema.to_vec(), vec![OrderType::descending(); c.schema.len()], @@ -146,7 +145,7 @@ fn memcmp_decode(c: &Case, bytes: &Vec>) -> Result>> { Ok(res) } -fn basic_decode(c: &Case, bytes: &Vec>) -> Result>> { +fn basic_decode(c: &Case, bytes: &Vec>) -> anyhow::Result>> { let table_columns = c .column_ids .iter() @@ -195,7 +194,7 @@ fn basic_decode(c: &Case, bytes: &Vec>) -> Result>> { Ok(res) } -fn column_aware_decode(c: &Case, bytes: &Vec>) -> Result>> { +fn column_aware_decode(c: &Case, bytes: &Vec>) -> anyhow::Result>> { let table_columns = c .column_ids .iter() diff --git a/src/storage/compactor/src/lib.rs b/src/storage/compactor/src/lib.rs index e95ba1ff6a39a..2204ffe0af7ef 100644 --- a/src/storage/compactor/src/lib.rs +++ b/src/storage/compactor/src/lib.rs @@ -49,6 +49,8 @@ pub struct CompactorOpts { #[clap(long, env = "RW_PORT")] pub port: Option, + /// We will start a http server at this address via `MetricsManager`. + /// Then the prometheus instance will poll the metrics from this address. #[clap( long, env = "RW_PROMETHEUS_LISTENER_ADDR", diff --git a/src/storage/compactor/src/rpc.rs b/src/storage/compactor/src/rpc.rs index d72e3de4f79ee..7990b2810f536 100644 --- a/src/storage/compactor/src/rpc.rs +++ b/src/storage/compactor/src/rpc.rs @@ -21,9 +21,10 @@ use risingwave_pb::compactor::{ }; use risingwave_pb::monitor_service::monitor_service_server::MonitorService; use risingwave_pb::monitor_service::{ - AnalyzeHeapRequest, AnalyzeHeapResponse, HeapProfilingRequest, HeapProfilingResponse, - ListHeapProfilingRequest, ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, - StackTraceRequest, StackTraceResponse, + AnalyzeHeapRequest, AnalyzeHeapResponse, GetBackPressureRequest, GetBackPressureResponse, + HeapProfilingRequest, HeapProfilingResponse, ListHeapProfilingRequest, + ListHeapProfilingResponse, ProfilingRequest, ProfilingResponse, StackTraceRequest, + StackTraceResponse, }; use tokio::sync::mpsc; use tonic::{Request, Response, Status}; @@ -133,4 +134,13 @@ impl MonitorService for MonitorServiceImpl { "Heap profiling unimplemented in compactor", )) } + + async fn get_back_pressure( + &self, + _request: Request, + ) -> Result, Status> { + Err(Status::unimplemented( + "Get Back Pressure unimplemented in compactor", + )) + } } diff --git a/src/storage/compactor/src/server.rs b/src/storage/compactor/src/server.rs index 756c6ae2a95c0..6d2c01527ceb9 100644 --- a/src/storage/compactor/src/server.rs +++ b/src/storage/compactor/src/server.rs @@ -23,7 +23,7 @@ use risingwave_common::config::{ }; use risingwave_common::monitor::connection::{RouterExt, TcpConfig}; use risingwave_common::system_param::local_manager::LocalSystemParamsManager; -use risingwave_common::system_param::reader::SystemParamsReader; +use risingwave_common::system_param::reader::{SystemParamsRead, SystemParamsReader}; use risingwave_common::telemetry::manager::TelemetryManager; use risingwave_common::telemetry::telemetry_env_enabled; use risingwave_common::util::addr::HostAddr; @@ -304,12 +304,12 @@ pub async fn compactor_serve( _ = tokio::signal::ctrl_c() => {}, _ = &mut shutdown_recv => { for (join_handle, shutdown_sender) in sub_tasks { - if let Err(err) = shutdown_sender.send(()) { - tracing::warn!("Failed to send shutdown: {:?}", err); + if shutdown_sender.send(()).is_err() { + tracing::warn!("Failed to send shutdown"); continue; } - if let Err(err) = join_handle.await { - tracing::warn!("Failed to join shutdown: {:?}", err); + if join_handle.await.is_err() { + tracing::warn!("Failed to join shutdown"); } } }, @@ -414,12 +414,12 @@ pub async fn shared_compactor_serve( tokio::select! { _ = tokio::signal::ctrl_c() => {}, _ = &mut shutdown_recv => { - if let Err(err) = shutdown_sender.send(()) { - tracing::warn!("Failed to send shutdown: {:?}", err); - } - if let Err(err) = join_handle.await { - tracing::warn!("Failed to join shutdown: {:?}", err); - } + if shutdown_sender.send(()).is_err() { + tracing::warn!("Failed to send shutdown"); + } + if join_handle.await.is_err() { + tracing::warn!("Failed to join shutdown"); + } }, } }, diff --git a/src/storage/hummock_sdk/Cargo.toml b/src/storage/hummock_sdk/Cargo.toml index b8f9c1e99499f..557aa4c8c8a54 100644 --- a/src/storage/hummock_sdk/Cargo.toml +++ b/src/storage/hummock_sdk/Cargo.toml @@ -18,7 +18,7 @@ bytes = "1" easy-ext = "1" hex = "0.4" itertools = "0.12" -parse-display = "0.8" +parse-display = "0.9" prost = { workspace = true } risingwave_common = { workspace = true } risingwave_pb = { workspace = true } diff --git a/src/storage/hummock_sdk/src/compact.rs b/src/storage/hummock_sdk/src/compact.rs index b607b167ebb9f..deb0b90bfcd2a 100644 --- a/src/storage/hummock_sdk/src/compact.rs +++ b/src/storage/hummock_sdk/src/compact.rs @@ -20,9 +20,10 @@ pub fn compact_task_to_string(compact_task: &CompactTask) -> String { let mut s = String::new(); writeln!( s, - "Compaction task id: {:?}, group-id: {:?}, target level: {:?}, target sub level: {:?}", + "Compaction task id: {:?}, group-id: {:?}, task type: {:?}, target level: {:?}, target sub level: {:?}", compact_task.task_id, compact_task.compaction_group_id, + compact_task.task_type(), compact_task.target_level, compact_task.target_sub_level_id ) @@ -163,7 +164,7 @@ pub fn estimate_memory_for_compact_task( // to the specified block and build the iterator. Since this operation is concurrent, the memory // usage will need to take into account the size of the SstableMeta. // The common size of SstableMeta in tests is no more than 1m (mainly from xor filters). - let mut max_meta_ratio = 0; + let mut task_max_sst_meta_ratio = 0; // The memory usage of the SstableStreamIterator comes from SstableInfo with some state // information (use ESTIMATED_META_SIZE to estimate it), the BlockStream being read (one block), @@ -173,24 +174,27 @@ pub fn estimate_memory_for_compact_task( // input for level in &task.input_ssts { if level.level_type() == LevelType::Nonoverlapping { - let mut meta_size = 0; + let mut cur_level_max_sst_meta_size = 0; for sst in &level.table_infos { - meta_size = std::cmp::max(meta_size, sst.file_size - sst.meta_offset); - max_meta_ratio = std::cmp::max(max_meta_ratio, meta_size * 100 / sst.file_size); + let meta_size = sst.file_size - sst.meta_offset; + task_max_sst_meta_ratio = + std::cmp::max(task_max_sst_meta_ratio, meta_size * 100 / sst.file_size); + cur_level_max_sst_meta_size = std::cmp::max(meta_size, cur_level_max_sst_meta_size); } - result += max_input_stream_estimated_memory + meta_size; + result += max_input_stream_estimated_memory + cur_level_max_sst_meta_size; } else { for sst in &level.table_infos { let meta_size = sst.file_size - sst.meta_offset; result += max_input_stream_estimated_memory + meta_size; - max_meta_ratio = std::cmp::max(max_meta_ratio, meta_size * 100 / sst.file_size); + task_max_sst_meta_ratio = + std::cmp::max(task_max_sst_meta_ratio, meta_size * 100 / sst.file_size); } } } // output // builder will maintain SstableInfo + block_builder(block) + writer (block to vec) - let estimated_meta_size = sst_capacity * max_meta_ratio / 100; + let estimated_meta_size = sst_capacity * task_max_sst_meta_ratio / 100; if support_streaming_upload { result += estimated_meta_size + 2 * block_size } else { diff --git a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs index a6950c909e958..9e07598d07920 100644 --- a/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs +++ b/src/storage/hummock_sdk/src/compaction_group/hummock_version_ext.rs @@ -668,6 +668,13 @@ impl Levels { &mut self.levels[level_idx - 1] } + pub fn is_last_level(&self, level_idx: u32) -> bool { + self.levels + .last() + .as_ref() + .map_or(false, |level| level.level_idx == level_idx) + } + pub fn count_ssts(&self) -> usize { self.get_level0() .get_sub_levels() diff --git a/src/storage/hummock_sdk/src/key.rs b/src/storage/hummock_sdk/src/key.rs index 26f34fa1f81bd..c9ef17693f818 100644 --- a/src/storage/hummock_sdk/src/key.rs +++ b/src/storage/hummock_sdk/src/key.rs @@ -15,6 +15,7 @@ use std::borrow::Borrow; use std::cmp::Ordering; use std::fmt::Debug; +use std::iter::once; use std::ops::Bound::*; use std::ops::{Bound, Deref, DerefMut, RangeBounds}; use std::ptr; @@ -652,10 +653,11 @@ impl> Debug for FullKey { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( f, - "FullKey {{ {:?}, epoch: {}, epoch_with_gap: {}}}", + "FullKey {{ {:?}, epoch: {}, epoch_with_gap: {}, spill_offset: {}}}", self.user_key, + self.epoch_with_gap.pure_epoch(), self.epoch_with_gap.as_u64(), - self.epoch_with_gap.pure_epoch() + self.epoch_with_gap.as_u64() - self.epoch_with_gap.pure_epoch(), ) } } @@ -826,6 +828,15 @@ impl + Ord + Eq> PartialOrd for FullKey { } } +impl<'a, T> From> for UserKey +where + T: AsRef<[u8]> + CopyFromSlice, +{ + fn from(value: UserKey<&'a [u8]>) -> Self { + value.copy_into() + } +} + #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct PointRange> { // When comparing `PointRange`, we first compare `left_user_key`, then @@ -921,6 +932,143 @@ pub fn bound_table_key_range + EmptySliceRef>( (start, end) } +/// TODO: Temporary bypass full key check. Remove this field after #15099 is resolved. +pub struct FullKeyTracker + Ord + Eq, const SKIP_DEDUP: bool = false> { + pub latest_full_key: FullKey, + last_observed_epoch_with_gap: EpochWithGap, +} + +impl + Ord + Eq, const SKIP_DEDUP: bool> FullKeyTracker { + pub fn new(init_full_key: FullKey) -> Self { + let epoch_with_gap = init_full_key.epoch_with_gap; + Self { + latest_full_key: init_full_key, + last_observed_epoch_with_gap: epoch_with_gap, + } + } + + /// Check and observe a new full key during iteration + /// + /// # Examples: + /// ``` + /// use bytes::Bytes; + /// use risingwave_common::catalog::TableId; + /// use risingwave_common::util::epoch::EPOCH_AVAILABLE_BITS; + /// use risingwave_hummock_sdk::EpochWithGap; + /// use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker, TableKey}; + /// + /// let table_id = TableId { table_id: 1 }; + /// let full_key1 = FullKey::new(table_id, TableKey(Bytes::from("c")), 5 << EPOCH_AVAILABLE_BITS); + /// let mut a: FullKeyTracker<_> = FullKeyTracker::::new(full_key1.clone()); + /// + /// // Panic on non-decreasing epoch observed for the same user key. + /// // let full_key_with_larger_epoch = FullKey::new(table_id, TableKey(Bytes::from("c")), 6 << EPOCH_AVAILABLE_BITS); + /// // a.observe(full_key_with_larger_epoch); + /// + /// // Panic on non-increasing user key observed. + /// // let full_key_with_smaller_user_key = FullKey::new(table_id, TableKey(Bytes::from("b")), 3 << EPOCH_AVAILABLE_BITS); + /// // a.observe(full_key_with_smaller_user_key); + /// + /// let full_key2 = FullKey::new(table_id, TableKey(Bytes::from("c")), 3 << EPOCH_AVAILABLE_BITS); + /// assert_eq!(a.observe(full_key2.clone()), None); + /// assert_eq!(a.latest_user_key(), &full_key2.user_key); + /// + /// let full_key3 = FullKey::new(table_id, TableKey(Bytes::from("f")), 4 << EPOCH_AVAILABLE_BITS); + /// assert_eq!(a.observe(full_key3.clone()), Some(full_key1.user_key)); + /// assert_eq!(a.latest_user_key(), &full_key3.user_key); + /// ``` + /// + /// Return: + /// - If the provided `key` contains a new user key, return the latest full key observed for the previous user key. + /// - Otherwise: return None + pub fn observe(&mut self, key: FullKey) -> Option> + where + UserKey: Into>, + F: AsRef<[u8]>, + { + self.observe_multi_version(key.user_key, once(key.epoch_with_gap)) + } + + /// `epochs` comes from greater to smaller + pub fn observe_multi_version( + &mut self, + user_key: UserKey, + mut epochs: impl Iterator, + ) -> Option> + where + UserKey: Into>, + F: AsRef<[u8]>, + { + let max_epoch_with_gap = epochs.next().expect("non-empty"); + let min_epoch_with_gap = epochs.fold( + max_epoch_with_gap, + |prev_epoch_with_gap, curr_epoch_with_gap| { + assert!( + prev_epoch_with_gap > curr_epoch_with_gap, + "epoch list not sorted. prev: {:?}, curr: {:?}, user_key: {:?}", + prev_epoch_with_gap, + curr_epoch_with_gap, + user_key + ); + curr_epoch_with_gap + }, + ); + match self + .latest_full_key + .user_key + .as_ref() + .cmp(&user_key.as_ref()) + { + Ordering::Less => { + // Observe a new user key + + // Reset epochs + self.last_observed_epoch_with_gap = min_epoch_with_gap; + + // Take the previous key and set latest key + Some( + std::mem::replace( + &mut self.latest_full_key, + FullKey { + user_key: user_key.into(), + epoch_with_gap: min_epoch_with_gap, + }, + ) + .user_key, + ) + } + Ordering::Equal => { + if max_epoch_with_gap > self.last_observed_epoch_with_gap + || (!SKIP_DEDUP && max_epoch_with_gap == self.last_observed_epoch_with_gap) + { + // Epoch from the same user key should be monotonically decreasing + panic!( + "key {:?} epoch {:?} >= prev epoch {:?}", + user_key, max_epoch_with_gap, self.last_observed_epoch_with_gap + ); + } + self.last_observed_epoch_with_gap = min_epoch_with_gap; + None + } + Ordering::Greater => { + // User key should be monotonically increasing + panic!( + "key {:?} <= prev key {:?}", + user_key, + FullKey { + user_key: self.latest_full_key.user_key.as_ref(), + epoch_with_gap: self.last_observed_epoch_with_gap + } + ); + } + } + } + + pub fn latest_user_key(&self) -> &UserKey { + &self.latest_full_key.user_key + } +} + #[cfg(test)] mod tests { use std::cmp::Ordering; diff --git a/src/storage/hummock_sdk/src/lib.rs b/src/storage/hummock_sdk/src/lib.rs index b87daf70ee3c4..8df27676591dd 100644 --- a/src/storage/hummock_sdk/src/lib.rs +++ b/src/storage/hummock_sdk/src/lib.rs @@ -264,11 +264,16 @@ pub fn can_concat(ssts: &[SstableInfo]) -> bool { const CHECKPOINT_DIR: &str = "checkpoint"; const CHECKPOINT_NAME: &str = "0"; +const ARCHIVE_DIR: &str = "archive"; pub fn version_checkpoint_path(root_dir: &str) -> String { format!("{}/{}/{}", root_dir, CHECKPOINT_DIR, CHECKPOINT_NAME) } +pub fn version_archive_dir(root_dir: &str) -> String { + format!("{}/{}", root_dir, ARCHIVE_DIR) +} + pub fn version_checkpoint_dir(checkpoint_path: &str) -> String { checkpoint_path.trim_end_matches(|c| c != '/').to_string() } diff --git a/src/storage/hummock_sdk/src/table_watermark.rs b/src/storage/hummock_sdk/src/table_watermark.rs index cf6c41e698af5..76e2df56c4633 100644 --- a/src/storage/hummock_sdk/src/table_watermark.rs +++ b/src/storage/hummock_sdk/src/table_watermark.rs @@ -282,6 +282,15 @@ pub enum WatermarkDirection { Descending, } +impl ToString for WatermarkDirection { + fn to_string(&self) -> String { + match self { + WatermarkDirection::Ascending => "Ascending".to_string(), + WatermarkDirection::Descending => "Descending".to_string(), + } + } +} + impl WatermarkDirection { pub fn filter_by_watermark(&self, key: impl AsRef<[u8]>, watermark: impl AsRef<[u8]>) -> bool { let key = key.as_ref(); @@ -340,8 +349,8 @@ impl VnodeWatermark { #[derive(Clone, Debug, PartialEq)] pub struct TableWatermarks { // later epoch at the back - pub(crate) watermarks: Vec<(HummockEpoch, Vec)>, - pub(crate) direction: WatermarkDirection, + pub watermarks: Vec<(HummockEpoch, Vec)>, + pub direction: WatermarkDirection, } impl TableWatermarks { diff --git a/src/storage/hummock_test/benches/bench_hummock_iter.rs b/src/storage/hummock_test/benches/bench_hummock_iter.rs index 0ddfe02b9438e..7b1cd9d260b85 100644 --- a/src/storage/hummock_test/benches/bench_hummock_iter.rs +++ b/src/storage/hummock_test/benches/bench_hummock_iter.rs @@ -92,7 +92,6 @@ fn criterion_benchmark(c: &mut Criterion) { runtime .block_on(hummock_storage.ingest_batch( batch, - vec![], WriteOptions { epoch, table_id: Default::default(), diff --git a/src/storage/hummock_test/src/bin/replay/main.rs b/src/storage/hummock_test/src/bin/replay/main.rs index e8637de487734..276a72bb26592 100644 --- a/src/storage/hummock_test/src/bin/replay/main.rs +++ b/src/storage/hummock_test/src/bin/replay/main.rs @@ -16,6 +16,9 @@ #![feature(coroutines)] #![feature(stmt_expr_attributes)] #![feature(proc_macro_hygiene)] +#![feature(register_tool)] +#![register_tool(rw)] +#![allow(rw::format_error)] // test code #[macro_use] mod replay_impl; @@ -40,7 +43,7 @@ use risingwave_object_store::object::build_remote_object_store; use risingwave_storage::filter_key_extractor::{ FakeRemoteTableAccessor, RpcFilterKeyExtractorManager, }; -use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore}; +use risingwave_storage::hummock::{FileCache, HummockStorage, SstableStore, SstableStoreConfig}; use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics, ObjectStoreMetrics}; use risingwave_storage::opts::StorageOpts; use serde::{Deserialize, Serialize}; @@ -94,33 +97,32 @@ async fn create_replay_hummock(r: Record, args: &Args) -> Result Result Result<()> { - self.store - .clear_shared_buffer() - .await - .map_err(|_| TraceError::ClearSharedBufferFailed)?; - Ok(()) + async fn clear_shared_buffer(&self, prev_epoch: u64) { + self.store.clear_shared_buffer(prev_epoch).await } } pub(crate) struct LocalReplayImpl(LocalHummockStorage); @@ -213,23 +209,20 @@ impl LocalReplay for LocalReplayImpl { self.0.epoch() } - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> Result { - let delete_ranges = delete_ranges - .into_iter() - .map(|(start, end)| (start.map(TracedBytes::into), end.map(TracedBytes::into))) - .collect(); - self.0 - .flush(delete_ranges) - .await - .map_err(|_| TraceError::FlushFailed) + async fn flush(&mut self) -> Result { + self.0.flush().await.map_err(|_| TraceError::FlushFailed) } fn is_dirty(&self) -> bool { self.0.is_dirty() } + + async fn try_flush(&mut self) -> Result<()> { + self.0 + .try_flush() + .await + .map_err(|_| TraceError::TryFlushFailed) + } } #[async_trait::async_trait] diff --git a/src/storage/hummock_test/src/compactor_tests.rs b/src/storage/hummock_test/src/compactor_tests.rs index 7759244f9f085..612d2b15e1c51 100644 --- a/src/storage/hummock_test/src/compactor_tests.rs +++ b/src/storage/hummock_test/src/compactor_tests.rs @@ -23,18 +23,25 @@ pub(crate) mod tests { use bytes::{BufMut, Bytes, BytesMut}; use itertools::Itertools; use rand::{Rng, RngCore, SeedableRng}; + use risingwave_common::buffer::BitmapBuilder; use risingwave_common::cache::CachePriority; use risingwave_common::catalog::TableId; use risingwave_common::constants::hummock::CompactionFilterFlag; + use risingwave_common::hash::VirtualNode; use risingwave_common::util::epoch::{ test_epoch, Epoch, EpochExt, EPOCH_INC_MIN_STEP_FOR_TEST, }; use risingwave_common_service::observer_manager::NotificationClient; use risingwave_hummock_sdk::can_concat; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; - use risingwave_hummock_sdk::key::{next_key, FullKey, TableKey, TABLE_PREFIX_LEN}; + use risingwave_hummock_sdk::key::{ + next_key, prefix_slice_with_vnode, FullKey, TableKey, TABLE_PREFIX_LEN, + }; use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; + use risingwave_hummock_sdk::table_watermark::{ + ReadTableWatermark, VnodeWatermark, WatermarkDirection, + }; use risingwave_hummock_sdk::version::HummockVersion; use risingwave_meta::hummock::compaction::compaction_config::CompactionConfigBuilder; use risingwave_meta::hummock::compaction::selector::{ @@ -46,7 +53,10 @@ pub(crate) mod tests { }; use risingwave_meta::hummock::{HummockManagerRef, MockHummockMetaClient}; use risingwave_pb::common::{HostAddress, WorkerType}; - use risingwave_pb::hummock::{CompactTask, InputLevel, KeyRange, SstableInfo, TableOption}; + use risingwave_pb::hummock::table_watermarks::PbEpochNewWatermarks; + use risingwave_pb::hummock::{ + CompactTask, InputLevel, KeyRange, SstableInfo, TableOption, TableWatermarks, + }; use risingwave_pb::meta::add_worker_node_request::Property; use risingwave_rpc_client::HummockMetaClient; use risingwave_storage::filter_key_extractor::{ @@ -59,7 +69,9 @@ pub(crate) mod tests { CompactionExecutor, CompactorContext, DummyCompactionFilter, TaskProgress, }; use risingwave_storage::hummock::iterator::test_utils::mock_sstable_store; - use risingwave_storage::hummock::iterator::{ConcatIterator, UserIterator}; + use risingwave_storage::hummock::iterator::{ + ConcatIterator, SkipWatermarkIterator, UserIterator, + }; use risingwave_storage::hummock::sstable_store::SstableStoreRef; use risingwave_storage::hummock::test_utils::gen_test_sstable_info; use risingwave_storage::hummock::value::HummockValue; @@ -161,7 +173,6 @@ pub(crate) mod tests { TableKey(key.clone()), StorageValue::new_put(Bytes::from(new_val)), )], - vec![], WriteOptions { epoch, table_id: Default::default(), @@ -297,13 +308,13 @@ pub(crate) mod tests { compact_task.table_options = BTreeMap::from([( 0, TableOption { - retention_seconds: 64, + retention_seconds: Some(64), }, )]); compact_task.current_epoch_time = 0; let (_tx, rx) = tokio::sync::oneshot::channel(); - let (result_task, task_stats) = compact( + let ((result_task, task_stats), _) = compact( compact_ctx.clone(), compact_task.clone(), rx, @@ -359,7 +370,7 @@ pub(crate) mod tests { .sstable(&output_sst, &mut StoreLocalStatistic::default()) .await .unwrap(); - table_key_count += table.value().meta.key_count; + table_key_count += table.meta.key_count; } // we have removed these 31 keys before watermark 32. @@ -459,7 +470,7 @@ pub(crate) mod tests { { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (result_task, task_stats) = compact( + let ((result_task, task_stats), _) = compact( compact_ctx.clone(), compact_task.clone(), rx, @@ -495,9 +506,9 @@ pub(crate) mod tests { .unwrap(); let target_table_size = storage.storage_opts().sstable_size_mb * (1 << 20); assert!( - table.value().meta.estimated_size > target_table_size, + table.meta.estimated_size > target_table_size, "table.meta.estimated_size {} <= target_table_size {}", - table.value().meta.estimated_size, + table.meta.estimated_size, target_table_size ); } @@ -562,7 +573,7 @@ pub(crate) mod tests { .insert(TableKey(Bytes::from(key)), val.clone(), None) .unwrap(); } - local.flush(Vec::new()).await.unwrap(); + local.flush().await.unwrap(); let next_epoch = epoch.next_epoch(); local.seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test()); @@ -748,7 +759,7 @@ pub(crate) mod tests { storage .insert(TableKey(prefix.freeze()), val.clone(), None) .unwrap(); - storage.flush(Vec::new()).await.unwrap(); + storage.flush().await.unwrap(); storage.seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test()); other.seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test()); @@ -792,7 +803,7 @@ pub(crate) mod tests { // 4. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (result_task, task_stats) = compact( + let ((result_task, task_stats), _) = compact( compact_ctx, compact_task.clone(), rx, @@ -826,7 +837,6 @@ pub(crate) mod tests { .sstable(&table, &mut StoreLocalStatistic::default()) .await .unwrap() - .value() .meta .key_count; } @@ -939,7 +949,7 @@ pub(crate) mod tests { local .insert(TableKey(prefix.freeze()), val.clone(), None) .unwrap(); - local.flush(Vec::new()).await.unwrap(); + local.flush().await.unwrap(); local.seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test()); let ssts = storage @@ -970,7 +980,7 @@ pub(crate) mod tests { compact_task.table_options = BTreeMap::from_iter([( existing_table_id, TableOption { - retention_seconds: retention_seconds_expire_second, + retention_seconds: Some(retention_seconds_expire_second), }, )]); compact_task.current_epoch_time = epoch; @@ -987,7 +997,7 @@ pub(crate) mod tests { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (result_task, task_stats) = compact( + let ((result_task, task_stats), _) = compact( compact_ctx, compact_task.clone(), rx, @@ -1021,7 +1031,6 @@ pub(crate) mod tests { .sstable(&table, &mut StoreLocalStatistic::default()) .await .unwrap() - .value() .meta .key_count; } @@ -1137,7 +1146,7 @@ pub(crate) mod tests { local .insert(TableKey(Bytes::from(ramdom_key)), val.clone(), None) .unwrap(); - local.flush(Vec::new()).await.unwrap(); + local.flush().await.unwrap(); local.seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test()); let ssts = storage .seal_and_sync_epoch(epoch) @@ -1176,7 +1185,7 @@ pub(crate) mod tests { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (result_task, task_stats) = compact( + let ((result_task, task_stats), _) = compact( compact_ctx, compact_task.clone(), rx, @@ -1211,7 +1220,6 @@ pub(crate) mod tests { .sstable(table, &mut StoreLocalStatistic::default()) .await .unwrap() - .value() .meta .key_count; } @@ -1268,6 +1276,9 @@ pub(crate) mod tests { assert_eq!(key_count, scan_count); } + #[ignore] + // may update the test after https://github.com/risingwavelabs/risingwave/pull/14320 is merged. + // This PR will support #[tokio::test] async fn test_compaction_delete_range() { let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) = @@ -1299,17 +1310,20 @@ pub(crate) mod tests { .await; let epoch = test_epoch(130); local.init_for_test(epoch).await.unwrap(); - let prefix_key_range = |k: u16| { - let key = k.to_be_bytes(); - ( - Bound::Included(Bytes::copy_from_slice(key.as_slice())), - Bound::Excluded(Bytes::copy_from_slice(next_key(key.as_slice()).as_slice())), - ) - }; - local - .flush(vec![prefix_key_range(1u16), prefix_key_range(2u16)]) - .await - .unwrap(); + + local.flush().await.unwrap(); + // TODO: seal with table watermark like the following code + // let prefix_key_range = |k: u16| { + // let key = k.to_be_bytes(); + // ( + // Bound::Included(Bytes::copy_from_slice(key.as_slice())), + // Bound::Excluded(Bytes::copy_from_slice(next_key(key.as_slice()).as_slice())), + // ) + // }; + // local + // .flush(vec![prefix_key_range(1u16), prefix_key_range(2u16)]) + // .await + // .unwrap(); local.seal_current_epoch(u64::MAX, SealCurrentEpochOptions::for_test()); flush_and_commit(&hummock_meta_client, &storage, epoch).await; @@ -1342,7 +1356,7 @@ pub(crate) mod tests { // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (result_task, task_stats) = compact( + let ((result_task, task_stats), _) = compact( compact_ctx, compact_task.clone(), rx, @@ -1369,7 +1383,7 @@ pub(crate) mod tests { .last() .unwrap(); assert_eq!(1, output_level_info.table_infos.len()); - assert_eq!(254, output_level_info.table_infos[0].total_key_count); + assert_eq!(252, output_level_info.table_infos[0].total_key_count); } type KeyValue = (FullKey>, HummockValue>); @@ -1435,14 +1449,10 @@ pub(crate) mod tests { assert_eq!(normal_iter.value(), fast_iter.value()); let key_ref = fast_iter.key().user_key.as_ref(); assert!(normal_tables.iter().any(|table| { - table - .value() - .may_match_hash(&(Bound::Included(key_ref), Bound::Included(key_ref)), hash) + table.may_match_hash(&(Bound::Included(key_ref), Bound::Included(key_ref)), hash) })); assert!(fast_tables.iter().any(|table| { - table - .value() - .may_match_hash(&(Bound::Included(key_ref), Bound::Included(key_ref)), hash) + table.may_match_hash(&(Bound::Included(key_ref), Bound::Included(key_ref)), hash) })); normal_iter.next().await.unwrap(); fast_iter.next().await.unwrap(); @@ -1450,6 +1460,44 @@ pub(crate) mod tests { } } + async fn run_fast_and_normal_runner( + compact_ctx: CompactorContext, + task: CompactTask, + ) -> (Vec, Vec) { + let multi_filter_key_extractor = + Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)); + let compaction_filter = DummyCompactionFilter {}; + let slow_compact_runner = CompactorRunner::new( + 0, + compact_ctx.clone(), + task.clone(), + Box::new(SharedComapctorObjectIdManager::for_test( + VecDeque::from_iter([5, 6, 7, 8, 9, 10, 11, 12, 13]), + )), + ); + let fast_compact_runner = FastCompactorRunner::new( + compact_ctx.clone(), + task.clone(), + multi_filter_key_extractor.clone(), + Box::new(SharedComapctorObjectIdManager::for_test( + VecDeque::from_iter([22, 23, 24, 25, 26, 27, 28, 29]), + )), + Arc::new(TaskProgress::default()), + ); + let (_, ret1, _) = slow_compact_runner + .run( + compaction_filter, + multi_filter_key_extractor, + Arc::new(TaskProgress::default()), + ) + .await + .unwrap(); + let ret = ret1.into_iter().map(|sst| sst.sst_info).collect_vec(); + let (ssts, _) = fast_compact_runner.run().await.unwrap(); + let fast_ret = ssts.into_iter().map(|sst| sst.sst_info).collect_vec(); + (ret, fast_ret) + } + async fn test_fast_compact_impl(data: Vec>) { let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) = setup_compute_env(8080).await; @@ -1517,37 +1565,7 @@ pub(crate) mod tests { gc_delete_keys: true, ..Default::default() }; - let multi_filter_key_extractor = - Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)); - let compaction_filter = DummyCompactionFilter {}; - let slow_compact_runner = CompactorRunner::new( - 0, - compact_ctx.clone(), - task.clone(), - Box::new(SharedComapctorObjectIdManager::for_test( - VecDeque::from_iter([5, 6, 7, 8, 9, 10, 11, 12, 13]), - )), - ); - let fast_compact_runner = FastCompactorRunner::new( - compact_ctx.clone(), - task.clone(), - multi_filter_key_extractor.clone(), - Box::new(SharedComapctorObjectIdManager::for_test( - VecDeque::from_iter([22, 23, 24, 25, 26, 27, 28, 29]), - )), - Arc::new(TaskProgress::default()), - ); - let (_, ret1, _) = slow_compact_runner - .run( - compaction_filter, - multi_filter_key_extractor, - Arc::new(TaskProgress::default()), - ) - .await - .unwrap(); - let ret = ret1.into_iter().map(|sst| sst.sst_info).collect_vec(); - let (ssts, _) = fast_compact_runner.run().await.unwrap(); - let fast_ret = ssts.into_iter().map(|sst| sst.sst_info).collect_vec(); + let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await; check_compaction_result(compact_ctx.sstable_store, ret, fast_ret, capacity).await; } @@ -1769,37 +1787,215 @@ pub(crate) mod tests { gc_delete_keys: true, ..Default::default() }; - let multi_filter_key_extractor = - Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)); - let compaction_filter = DummyCompactionFilter {}; - let slow_compact_runner = CompactorRunner::new( - 0, - compact_ctx.clone(), - task.clone(), - Box::new(SharedComapctorObjectIdManager::for_test( - VecDeque::from_iter([5, 6, 7, 8, 9, 10, 11, 12, 13]), - )), + let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await; + check_compaction_result(compact_ctx.sstable_store, ret, fast_ret, target_file_size).await; + } + + #[tokio::test] + async fn test_skip_watermark() { + let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) = + setup_compute_env(8080).await; + let hummock_meta_client: Arc = Arc::new(MockHummockMetaClient::new( + hummock_manager_ref.clone(), + worker_node.id, + )); + let existing_table_id: u32 = 1; + let storage = get_hummock_storage( + hummock_meta_client.clone(), + get_notification_client_for_test(env, hummock_manager_ref.clone(), worker_node.clone()), + &hummock_manager_ref, + TableId::from(existing_table_id), + ) + .await; + hummock_manager_ref.get_new_sst_ids(10).await.unwrap(); + let (compact_ctx, _) = prepare_compactor_and_filter(&storage, existing_table_id); + + let sstable_store = compact_ctx.sstable_store.clone(); + let capacity = 256 * 1024; + let opts = SstableBuilderOptions { + capacity, + block_capacity: 2048, + restart_interval: 16, + bloom_false_positive: 0.1, + compression_algorithm: CompressionAlgorithm::Lz4, + ..Default::default() + }; + + const KEY_COUNT: usize = 20000; + let mut rng = rand::rngs::StdRng::seed_from_u64( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), ); - let fast_compact_runner = FastCompactorRunner::new( - compact_ctx.clone(), - task.clone(), - multi_filter_key_extractor.clone(), - Box::new(SharedComapctorObjectIdManager::for_test( - VecDeque::from_iter([22, 23, 24, 25, 26, 27, 28, 29]), - )), - Arc::new(TaskProgress::default()), + let mut sst_infos = vec![]; + let mut max_sst_file_size = 0; + + for object_id in 1..3 { + let mut builder = SstableBuilder::<_, BlockedXor16FilterBuilder>::new( + object_id, + sstable_store + .clone() + .create_sst_writer(object_id, SstableWriterOptions::default()), + BlockedXor16FilterBuilder::create(opts.bloom_false_positive, opts.capacity / 16), + opts.clone(), + Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)), + None, + ); + let key_count = KEY_COUNT / VirtualNode::COUNT * 2; + for vnode_id in 0..VirtualNode::COUNT / 2 { + let mut last_k: u64 = 1; + let init_epoch = test_epoch(100 * object_id); + let mut last_epoch = init_epoch; + for idx in 0..key_count { + let rand_v = rng.next_u32() % 10; + let (k, epoch) = if rand_v == 0 { + (last_k + 1000 * object_id, init_epoch) + } else if rand_v < 5 { + (last_k, last_epoch.prev_epoch()) + } else { + (last_k + 1, init_epoch) + }; + let key = prefix_slice_with_vnode( + VirtualNode::from_index(vnode_id), + k.to_be_bytes().as_slice(), + ); + let key = FullKey::new(TableId::new(1), TableKey(key), epoch); + let rand_v = rng.next_u32() % 10; + let v = if (5..7).contains(&rand_v) { + HummockValue::delete() + } else { + HummockValue::put(format!("{}-{}", idx, epoch).into_bytes()) + }; + if rand_v < 5 && builder.current_block_size() > opts.block_capacity / 2 { + // cut block when the key is same with the last key. + builder.build_block().await.unwrap(); + } + builder.add(key.to_ref(), v.as_slice()).await.unwrap(); + last_k = k; + last_epoch = epoch; + } + } + + let output = builder.finish().await.unwrap(); + output.writer_output.await.unwrap().unwrap(); + let sst_info = output.sst_info.sst_info; + max_sst_file_size = std::cmp::max(max_sst_file_size, sst_info.file_size); + sst_infos.push(sst_info); + } + println!( + "input data: {}", + sst_infos.iter().map(|sst| sst.file_size).sum::(), ); - let (_, ret1, _) = slow_compact_runner - .run( - compaction_filter, - multi_filter_key_extractor, - Arc::new(TaskProgress::default()), - ) - .await - .unwrap(); - let ret = ret1.into_iter().map(|sst| sst.sst_info).collect_vec(); - let (ssts, _) = fast_compact_runner.run().await.unwrap(); - let fast_ret = ssts.into_iter().map(|sst| sst.sst_info).collect_vec(); - check_compaction_result(compact_ctx.sstable_store, ret, fast_ret, target_file_size).await; + + let target_file_size = max_sst_file_size / 4; + let mut table_watermarks = BTreeMap::default(); + let key_count = KEY_COUNT / VirtualNode::COUNT * 2; + let mut vnode_builder = BitmapBuilder::zeroed(VirtualNode::COUNT); + for i in 0..VirtualNode::COUNT / 2 { + if i % 2 == 0 { + vnode_builder.set(i, true); + } else { + vnode_builder.set(i, false); + } + } + let bitmap = Arc::new(vnode_builder.finish()); + let watermark_idx = key_count * 20; + let watermark_key = Bytes::from(watermark_idx.to_be_bytes().to_vec()); + table_watermarks.insert( + 1, + TableWatermarks { + epoch_watermarks: vec![PbEpochNewWatermarks { + watermarks: vec![ + VnodeWatermark::new(bitmap.clone(), watermark_key.clone()).to_protobuf() + ], + epoch: test_epoch(500), + }], + is_ascending: true, + }, + ); + + let task = CompactTask { + input_ssts: vec![ + InputLevel { + level_idx: 5, + level_type: 1, + table_infos: sst_infos.drain(..1).collect_vec(), + }, + InputLevel { + level_idx: 6, + level_type: 1, + table_infos: sst_infos, + }, + ], + existing_table_ids: vec![1], + task_id: 1, + watermark: 1000, + splits: vec![KeyRange::inf()], + target_level: 6, + base_level: 4, + target_file_size, + compression_algorithm: 1, + gc_delete_keys: true, + table_watermarks, + ..Default::default() + }; + let (ret, fast_ret) = run_fast_and_normal_runner(compact_ctx.clone(), task).await; + println!( + "normal compact result data: {}, fast compact result data: {}", + ret.iter().map(|sst| sst.file_size).sum::(), + fast_ret.iter().map(|sst| sst.file_size).sum::(), + ); + // check_compaction_result(compact_ctx.sstable_store, ret.clone(), fast_ret, target_file_size).await; + let mut fast_tables = Vec::with_capacity(fast_ret.len()); + let mut normal_tables = Vec::with_capacity(ret.len()); + let mut stats = StoreLocalStatistic::default(); + for sst_info in &fast_ret { + fast_tables.push(sstable_store.sstable(sst_info, &mut stats).await.unwrap()); + } + + for sst_info in &ret { + normal_tables.push(sstable_store.sstable(sst_info, &mut stats).await.unwrap()); + } + assert!(can_concat(&ret)); + assert!(can_concat(&fast_ret)); + let read_options = Arc::new(SstableIteratorReadOptions::default()); + + let mut watermark = ReadTableWatermark { + direction: WatermarkDirection::Ascending, + vnode_watermarks: BTreeMap::default(), + }; + for i in 0..VirtualNode::COUNT { + if i % 2 == 0 { + watermark + .vnode_watermarks + .insert(VirtualNode::from_index(i), watermark_key.clone()); + } + } + let watermark = BTreeMap::from_iter([(TableId::new(1), watermark)]); + + let mut normal_iter = UserIterator::for_test( + SkipWatermarkIterator::new( + ConcatIterator::new(ret, sstable_store.clone(), read_options.clone()), + watermark.clone(), + ), + (Bound::Unbounded, Bound::Unbounded), + ); + let mut fast_iter = UserIterator::for_test( + SkipWatermarkIterator::new( + ConcatIterator::new(fast_ret, sstable_store, read_options), + watermark, + ), + (Bound::Unbounded, Bound::Unbounded), + ); + normal_iter.rewind().await.unwrap(); + fast_iter.rewind().await.unwrap(); + let mut count = 0; + while normal_iter.is_valid() { + assert_eq!(normal_iter.key(), fast_iter.key(), "not equal in {}", count,); + normal_iter.next().await.unwrap(); + fast_iter.next().await.unwrap(); + count += 1; + } } } diff --git a/src/storage/hummock_test/src/failpoint_tests.rs b/src/storage/hummock_test/src/failpoint_tests.rs index 0b3ea152dd2d6..2b2f3da3f15a2 100644 --- a/src/storage/hummock_test/src/failpoint_tests.rs +++ b/src/storage/hummock_test/src/failpoint_tests.rs @@ -86,7 +86,6 @@ async fn test_failpoints_state_store_read_upload() { local .ingest_batch( batch1, - vec![], WriteOptions { epoch: 1, table_id: Default::default(), @@ -125,7 +124,6 @@ async fn test_failpoints_state_store_read_upload() { local .ingest_batch( batch2, - vec![], WriteOptions { epoch: 3, table_id: Default::default(), diff --git a/src/storage/hummock_test/src/hummock_read_version_tests.rs b/src/storage/hummock_test/src/hummock_read_version_tests.rs index b8370f15f1163..147353aee0d30 100644 --- a/src/storage/hummock_test/src/hummock_read_version_tests.rs +++ b/src/storage/hummock_test/src/hummock_read_version_tests.rs @@ -21,7 +21,7 @@ use parking_lot::RwLock; use risingwave_common::catalog::TableId; use risingwave_common::util::epoch::{test_epoch, EpochExt, EPOCH_INC_MIN_STEP_FOR_TEST}; use risingwave_hummock_sdk::key::{key_with_epoch, map_table_key_range}; -use risingwave_hummock_sdk::LocalSstableInfo; +use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; use risingwave_meta::hummock::test_utils::setup_compute_env; use risingwave_pb::hummock::{KeyRange, SstableInfo}; use risingwave_storage::hummock::iterator::test_utils::{ @@ -32,7 +32,7 @@ use risingwave_storage::hummock::store::version::{ read_filter_for_batch, read_filter_for_local, HummockReadVersion, StagingData, StagingSstableInfo, VersionUpdate, }; -use risingwave_storage::hummock::test_utils::gen_dummy_batch; +use risingwave_storage::hummock::test_utils::{gen_dummy_batch, gen_dummy_sst_info}; use crate::test_utils::prepare_first_valid_version; @@ -53,15 +53,12 @@ async fn test_read_version_basic() { let kv_pairs = gen_dummy_batch(1); let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs); let size = SharedBufferBatch::measure_batch_size(&sorted_items); - let imm = SharedBufferBatch::build_shared_buffer_batch( + let imm = SharedBufferBatch::build_shared_buffer_batch_for_test( epoch, 0, sorted_items, size, - vec![], TableId::from(table_id), - None, - None, ); read_version.update(VersionUpdate::Staging(StagingData::ImmMem(imm))); @@ -91,15 +88,12 @@ async fn test_read_version_basic() { let kv_pairs = gen_dummy_batch(i + 2); let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs); let size = SharedBufferBatch::measure_batch_size(&sorted_items); - let imm = SharedBufferBatch::build_shared_buffer_batch( + let imm = SharedBufferBatch::build_shared_buffer_batch_for_test( epoch, 0, sorted_items, size, - vec![], TableId::from(table_id), - None, - None, ); read_version.update(VersionUpdate::Staging(StagingData::ImmMem(imm))); @@ -280,15 +274,12 @@ async fn test_read_filter_basic() { let kv_pairs = gen_dummy_batch(epoch); let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs); let size = SharedBufferBatch::measure_batch_size(&sorted_items); - let imm = SharedBufferBatch::build_shared_buffer_batch( + let imm = SharedBufferBatch::build_shared_buffer_batch_for_test( epoch, 0, sorted_items, size, - vec![], TableId::from(table_id), - None, - None, ); read_version @@ -346,3 +337,69 @@ async fn test_read_filter_basic() { } } } + +#[tokio::test] +async fn test_read_filter_for_batch_issue_14659() { + use std::ops::Bound::Unbounded; + + let (env, hummock_manager_ref, _cluster_manager_ref, worker_node) = + setup_compute_env(8080).await; + + let (pinned_version, _, _) = + prepare_first_valid_version(env, hummock_manager_ref, worker_node).await; + + const NUM_SHARDS: u64 = 2; + let table_id = TableId::from(2); + let epoch = test_epoch(1); + let mut read_version_vec = vec![]; + let mut imms = vec![]; + + // Populate IMMs + for i in 0..NUM_SHARDS { + let read_version = Arc::new(RwLock::new(HummockReadVersion::new( + table_id, + pinned_version.clone(), + ))); + + let items = SharedBufferBatch::build_shared_buffer_item_batches(gen_dummy_batch(i)); + let size = SharedBufferBatch::measure_batch_size(&items); + let imm = + SharedBufferBatch::build_shared_buffer_batch_for_test(epoch, 0, items, size, table_id); + + imms.push(imm.clone()); + + read_version + .write() + .update(VersionUpdate::Staging(StagingData::ImmMem(imm))); + + read_version_vec.push(read_version); + } + + // Update read version via staging SSTs + let sst_id = 233; + let staging_sst = gen_dummy_sst_info(sst_id, imms.clone(), table_id, epoch); + read_version_vec.iter().for_each(|v| { + v.write().update(VersionUpdate::Staging(StagingData::Sst( + StagingSstableInfo::new( + vec![LocalSstableInfo::for_test(staging_sst.clone())], + vec![epoch], + imms.iter().map(|imm| imm.batch_id()).collect_vec(), + imms.iter().map(|imm| imm.size()).sum(), + ), + ))); + }); + + // build for batch with max epoch + let (_, hummock_read_snapshot) = read_filter_for_batch( + HummockEpoch::MAX, + table_id, + (Unbounded, Unbounded), + read_version_vec, + ) + .unwrap(); + + // No imms should be proivided + assert_eq!(0, hummock_read_snapshot.0.len()); + // Only 1 staging sst is provided + assert_eq!(1, hummock_read_snapshot.1.len()); +} diff --git a/src/storage/hummock_test/src/hummock_storage_tests.rs b/src/storage/hummock_test/src/hummock_storage_tests.rs index 64e9c7a947445..c84f65c9322b3 100644 --- a/src/storage/hummock_test/src/hummock_storage_tests.rs +++ b/src/storage/hummock_test/src/hummock_storage_tests.rs @@ -107,7 +107,6 @@ async fn test_storage_basic() { hummock_storage .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -170,7 +169,6 @@ async fn test_storage_basic() { hummock_storage .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -203,7 +201,6 @@ async fn test_storage_basic() { hummock_storage .ingest_batch( batch3, - vec![], WriteOptions { epoch: epoch3, table_id: TEST_TABLE_ID, @@ -475,7 +472,6 @@ async fn test_state_store_sync() { hummock_storage .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -503,7 +499,6 @@ async fn test_state_store_sync() { hummock_storage .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -524,7 +519,6 @@ async fn test_state_store_sync() { hummock_storage .ingest_batch( batch3, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -769,7 +763,6 @@ async fn test_delete_get() { hummock_storage .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -793,7 +786,6 @@ async fn test_delete_get() { hummock_storage .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -855,7 +847,6 @@ async fn test_multiple_epoch_sync() { hummock_storage .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -873,7 +864,6 @@ async fn test_multiple_epoch_sync() { hummock_storage .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -897,7 +887,6 @@ async fn test_multiple_epoch_sync() { hummock_storage .ingest_batch( batch3, - vec![], WriteOptions { epoch: epoch3, table_id: TEST_TABLE_ID, @@ -1006,7 +995,6 @@ async fn test_iter_with_min_epoch() { hummock_storage .ingest_batch( batch_epoch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -1025,7 +1013,6 @@ async fn test_iter_with_min_epoch() { hummock_storage .ingest_batch( batch_epoch2, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -1228,7 +1215,6 @@ async fn test_hummock_version_reader() { hummock_storage .ingest_batch( batch_epoch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -1241,7 +1227,6 @@ async fn test_hummock_version_reader() { hummock_storage .ingest_batch( batch_epoch2, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -1254,7 +1239,6 @@ async fn test_hummock_version_reader() { hummock_storage .ingest_batch( batch_epoch3, - vec![], WriteOptions { epoch: epoch3, table_id: TEST_TABLE_ID, @@ -1618,7 +1602,6 @@ async fn test_get_with_min_epoch() { hummock_storage .ingest_batch( batch_epoch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -1637,7 +1620,6 @@ async fn test_get_with_min_epoch() { hummock_storage .ingest_batch( batch_epoch2, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -1950,16 +1932,19 @@ async fn test_table_watermark() { (&mut local1, vnode_bitmap1.clone()), (&mut local2, vnode_bitmap2.clone()), ] { - local.flush(vec![]).await.unwrap(); + local.flush().await.unwrap(); local.seal_current_epoch( epoch2, - SealCurrentEpochOptions::new( - vec![VnodeWatermark::new( - Arc::new(vnode_bitmap), - gen_inner_key(watermark1), - )], - WatermarkDirection::Ascending, - ), + SealCurrentEpochOptions { + table_watermarks: Some(( + WatermarkDirection::Ascending, + vec![VnodeWatermark::new( + Arc::new(vnode_bitmap), + gen_inner_key(watermark1), + )], + )), + switch_op_consistency_level: None, + }, ); } @@ -2051,8 +2036,14 @@ async fn test_table_watermark() { for (key, value) in batch { local.insert(key, value, None).unwrap(); } - local.flush(vec![]).await.unwrap(); - local.seal_current_epoch(epoch3, SealCurrentEpochOptions::no_watermark()); + local.flush().await.unwrap(); + local.seal_current_epoch( + epoch3, + SealCurrentEpochOptions { + table_watermarks: None, + switch_op_consistency_level: None, + }, + ); } let indexes_after_epoch2 = || gen_range().filter(|index| index % 3 == 0 || index % 3 == 1); @@ -2293,13 +2284,16 @@ async fn test_table_watermark() { // regress watermark local.seal_current_epoch( epoch4, - SealCurrentEpochOptions::new( - vec![VnodeWatermark::new( - Arc::new(vnode_bitmap), - gen_inner_key(5), - )], - WatermarkDirection::Ascending, - ), + SealCurrentEpochOptions { + table_watermarks: Some(( + WatermarkDirection::Ascending, + vec![VnodeWatermark::new( + Arc::new(vnode_bitmap), + gen_inner_key(5), + )], + )), + switch_op_consistency_level: None, + }, ); } diff --git a/src/storage/hummock_test/src/local_state_store_test_utils.rs b/src/storage/hummock_test/src/local_state_store_test_utils.rs index 59472dec68b56..d9f2a503c39e3 100644 --- a/src/storage/hummock_test/src/local_state_store_test_utils.rs +++ b/src/storage/hummock_test/src/local_state_store_test_utils.rs @@ -13,16 +13,20 @@ // limitations under the License. use std::future::Future; +use std::sync::Arc; +use risingwave_common::buffer::Bitmap; +use risingwave_common::hash::VirtualNode; use risingwave_common::util::epoch::EpochPair; use risingwave_storage::error::StorageResult; use risingwave_storage::store::{InitOptions, LocalStateStore}; pub trait LocalStateStoreTestExt: LocalStateStore { fn init_for_test(&mut self, epoch: u64) -> impl Future> + Send + '_ { - self.init(InitOptions::new_with_epoch(EpochPair::new_test_epoch( - epoch, - ))) + self.init(InitOptions::new( + EpochPair::new_test_epoch(epoch), + Arc::new(Bitmap::ones(VirtualNode::COUNT)), + )) } } impl LocalStateStoreTestExt for T {} diff --git a/src/storage/hummock_test/src/snapshot_tests.rs b/src/storage/hummock_test/src/snapshot_tests.rs index 85de1f8234c23..09b6dffc07c40 100644 --- a/src/storage/hummock_test/src/snapshot_tests.rs +++ b/src/storage/hummock_test/src/snapshot_tests.rs @@ -125,7 +125,6 @@ async fn test_snapshot_inner( StorageValue::new_put("test"), ), ], - vec![], WriteOptions { epoch: epoch1, table_id: Default::default(), @@ -170,7 +169,6 @@ async fn test_snapshot_inner( StorageValue::new_put("test"), ), ], - vec![], WriteOptions { epoch: epoch2, table_id: Default::default(), @@ -216,7 +214,6 @@ async fn test_snapshot_inner( StorageValue::new_delete(), ), ], - vec![], WriteOptions { epoch: epoch3, table_id: Default::default(), @@ -279,7 +276,6 @@ async fn test_snapshot_range_scan_inner( StorageValue::new_put("test"), ), ], - vec![], WriteOptions { epoch, table_id: Default::default(), diff --git a/src/storage/hummock_test/src/state_store_tests.rs b/src/storage/hummock_test/src/state_store_tests.rs index 32c1536dda35e..594048d3fd58b 100644 --- a/src/storage/hummock_test/src/state_store_tests.rs +++ b/src/storage/hummock_test/src/state_store_tests.rs @@ -23,10 +23,7 @@ use risingwave_common::cache::CachePriority; use risingwave_common::catalog::{TableId, TableOption}; use risingwave_common::hash::VirtualNode; use risingwave_common::util::epoch::{test_epoch, EpochExt}; -use risingwave_hummock_sdk::key::FullKey; -use risingwave_hummock_sdk::{ - EpochWithGap, HummockReadEpoch, HummockSstableObjectId, LocalSstableInfo, -}; +use risingwave_hummock_sdk::{HummockReadEpoch, HummockSstableObjectId, LocalSstableInfo}; use risingwave_meta::hummock::test_utils::setup_compute_env; use risingwave_meta::hummock::MockHummockMetaClient; use risingwave_rpc_client::HummockMetaClient; @@ -135,7 +132,6 @@ async fn test_basic_inner( // try to write an empty batch, and hummock should write nothing let size = local .ingest_batch( - vec![], vec![], WriteOptions { epoch: epoch1, @@ -151,7 +147,6 @@ async fn test_basic_inner( local .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: Default::default(), @@ -209,7 +204,6 @@ async fn test_basic_inner( local .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch2, table_id: Default::default(), @@ -241,7 +235,6 @@ async fn test_basic_inner( local .ingest_batch( batch3, - vec![], WriteOptions { epoch: epoch3, table_id: Default::default(), @@ -440,7 +433,6 @@ async fn test_state_store_sync_inner( local .ingest_batch( batch1, - vec![], WriteOptions { epoch, table_id: Default::default(), @@ -468,7 +460,6 @@ async fn test_state_store_sync_inner( local .ingest_batch( batch2, - vec![], WriteOptions { epoch, table_id: Default::default(), @@ -498,7 +489,6 @@ async fn test_state_store_sync_inner( local .ingest_batch( batch3, - vec![], WriteOptions { epoch, table_id: Default::default(), @@ -541,7 +531,7 @@ async fn test_reload_storage() { let anchor = gen_key_from_str(VirtualNode::ZERO, "aa"); // First batch inserts the anchor and others. - let mut batch1 = vec![ + let mut batch1 = [ (anchor.clone(), StorageValue::new_put("111")), ( gen_key_from_str(VirtualNode::ZERO, "bb"), @@ -553,7 +543,7 @@ async fn test_reload_storage() { batch1.sort_by(|(k1, _), (k2, _)| k1.cmp(k2)); // Second batch modifies the anchor. - let mut batch2 = vec![ + let mut batch2 = [ ( gen_key_from_str(VirtualNode::ZERO, "cc"), StorageValue::new_put("333"), @@ -572,7 +562,6 @@ async fn test_reload_storage() { // hummock_storage // .ingest_batch( // batch1, - // vec![], // WriteOptions { // epoch: epoch1, // table_id: Default::default(), @@ -627,7 +616,6 @@ async fn test_reload_storage() { // hummock_storage // .ingest_batch( // batch2, - // vec![], // WriteOptions { // epoch: epoch2, // table_id: Default::default(), @@ -719,307 +707,305 @@ async fn test_reload_storage() { assert_eq!(len, 3); } -#[tokio::test] -async fn test_write_anytime_v2() { - let (hummock_storage, meta_client) = with_hummock_storage_v2(Default::default()).await; - test_write_anytime_inner(hummock_storage, meta_client).await; -} - -async fn test_write_anytime_inner( - hummock_storage: impl HummockStateStoreTestTrait, - _meta_client: Arc, -) { - let initial_epoch = hummock_storage.get_pinned_version().max_committed_epoch(); - - let epoch1 = test_epoch(initial_epoch.next_epoch()); - - let assert_old_value = |epoch: u64| { - let hummock_storage = &hummock_storage; - async move { - // check point get - assert_eq!( - "111".as_bytes(), - hummock_storage - .get( - gen_key_from_str(VirtualNode::ZERO, "aa"), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - } - ) - .await - .unwrap() - .unwrap() - ); - assert_eq!( - "222".as_bytes(), - hummock_storage - .get( - gen_key_from_str(VirtualNode::ZERO, "bb"), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - } - ) - .await - .unwrap() - .unwrap() - ); - assert_eq!( - "333".as_bytes(), - hummock_storage - .get( - gen_key_from_str(VirtualNode::ZERO, "cc"), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - } - ) - .await - .unwrap() - .unwrap() - ); - // check iter - let iter = hummock_storage - .iter( - ( - Bound::Included(gen_key_from_str(VirtualNode::ZERO, "aa")), - Bound::Included(gen_key_from_str(VirtualNode::ZERO, "cc")), - ), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - }, - ) - .await - .unwrap(); - futures::pin_mut!(iter); - assert_eq!( - ( - FullKey::new( - TableId::default(), - gen_key_from_str(VirtualNode::ZERO, "aa"), - epoch - ), - Bytes::from("111") - ), - iter.try_next().await.unwrap().unwrap() - ); - assert_eq!( - ( - FullKey::new( - TableId::default(), - gen_key_from_str(VirtualNode::ZERO, "bb"), - epoch - ), - Bytes::from("222") - ), - iter.try_next().await.unwrap().unwrap() - ); - assert_eq!( - ( - FullKey::new( - TableId::default(), - gen_key_from_str(VirtualNode::ZERO, "cc"), - epoch - ), - Bytes::from("333") - ), - iter.try_next().await.unwrap().unwrap() - ); - assert!(iter.try_next().await.unwrap().is_none()); - } - }; - - let batch1 = vec![ - ( - gen_key_from_str(VirtualNode::ZERO, "aa"), - StorageValue::new_put("111"), - ), - ( - gen_key_from_str(VirtualNode::ZERO, "bb"), - StorageValue::new_put("222"), - ), - ( - gen_key_from_str(VirtualNode::ZERO, "cc"), - StorageValue::new_put("333"), - ), - ]; - - let mut local = hummock_storage.new_local(NewLocalOptions::default()).await; - local.init_for_test(epoch1).await.unwrap(); - - local - .ingest_batch( - batch1.clone(), - vec![], - WriteOptions { - epoch: epoch1, - table_id: Default::default(), - }, - ) - .await - .unwrap(); - assert_old_value(epoch1).await; - - let assert_new_value = |epoch: u64| { - let hummock_storage = &hummock_storage; - async move { - // check point get - assert_eq!( - "111_new".as_bytes(), - hummock_storage - .get( - gen_key_from_str(VirtualNode::ZERO, "aa"), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - } - ) - .await - .unwrap() - .unwrap() - ); - - assert!(hummock_storage - .get( - gen_key_from_str(VirtualNode::ZERO, "bb"), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - } - ) - .await - .unwrap() - .is_none()); - assert_eq!( - "333".as_bytes(), - hummock_storage - .get( - gen_key_from_str(VirtualNode::ZERO, "cc"), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - } - ) - .await - .unwrap() - .unwrap() - ); - let iter = hummock_storage - .iter( - ( - Bound::Included(gen_key_from_str(VirtualNode::ZERO, "aa")), - Bound::Included(gen_key_from_str(VirtualNode::ZERO, "cc")), - ), - epoch, - ReadOptions { - cache_policy: CachePolicy::Fill(CachePriority::High), - ..Default::default() - }, - ) - .await - .unwrap(); - futures::pin_mut!(iter); - assert_eq!( - ( - FullKey::new_with_gap_epoch( - TableId::default(), - gen_key_from_str(VirtualNode::ZERO, "aa"), - EpochWithGap::new(epoch, 1) - ), - Bytes::from("111_new") - ), - iter.try_next().await.unwrap().unwrap() - ); - assert_eq!( - ( - FullKey::new_with_gap_epoch( - TableId::default(), - gen_key_from_str(VirtualNode::ZERO, "cc"), - EpochWithGap::new(epoch, 0) - ), - Bytes::from("333") - ), - iter.try_next().await.unwrap().unwrap() - ); - assert!(iter.try_next().await.unwrap().is_none()); - } - }; - - // Update aa, delete bb, cc unchanged - let batch2 = vec![ - ( - gen_key_from_str(VirtualNode::ZERO, "aa"), - StorageValue::new_put("111_new"), - ), - ( - gen_key_from_str(VirtualNode::ZERO, "bb"), - StorageValue::new_delete(), - ), - ]; - - local - .ingest_batch( - batch2, - vec![], - WriteOptions { - epoch: epoch1, - table_id: Default::default(), - }, - ) - .await - .unwrap(); - - assert_new_value(epoch1).await; - - let epoch2 = epoch1.next_epoch(); - local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test()); - - // Write to epoch2 - local - .ingest_batch( - batch1, - vec![], - WriteOptions { - epoch: epoch2, - table_id: Default::default(), - }, - ) - .await - .unwrap(); - local.seal_current_epoch(u64::MAX, SealCurrentEpochOptions::for_test()); - // Assert epoch 1 unchanged - assert_new_value(epoch1).await; - // Assert epoch 2 correctness - assert_old_value(epoch2).await; - - let ssts1 = hummock_storage - .seal_and_sync_epoch(epoch1) - .await - .unwrap() - .uncommitted_ssts; - assert_new_value(epoch1).await; - assert_old_value(epoch2).await; - - let ssts2 = hummock_storage - .seal_and_sync_epoch(epoch2) - .await - .unwrap() - .uncommitted_ssts; - assert_new_value(epoch1).await; - assert_old_value(epoch2).await; - - assert!(!ssts1.is_empty()); - assert!(!ssts2.is_empty()); -} +// Keep this test case's codes for future reference +// #[tokio::test] +// async fn test_write_anytime_v2() { +// let (hummock_storage, meta_client) = with_hummock_storage_v2(Default::default()).await; +// test_write_anytime_inner(hummock_storage, meta_client).await; +// } + +// async fn test_write_anytime_inner( +// hummock_storage: impl HummockStateStoreTestTrait, +// _meta_client: Arc, +// ) { +// let initial_epoch = hummock_storage.get_pinned_version().max_committed_epoch(); + +// let epoch1 = initial_epoch + 1; + +// let assert_old_value = |epoch| { +// let hummock_storage = &hummock_storage; +// async move { +// // check point get +// assert_eq!( +// "111".as_bytes(), +// hummock_storage +// .get( +// gen_key_from_str(VirtualNode::ZERO, "aa"), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// } +// ) +// .await +// .unwrap() +// .unwrap() +// ); +// assert_eq!( +// "222".as_bytes(), +// hummock_storage +// .get( +// gen_key_from_str(VirtualNode::ZERO, "bb"), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// } +// ) +// .await +// .unwrap() +// .unwrap() +// ); +// assert_eq!( +// "333".as_bytes(), +// hummock_storage +// .get( +// gen_key_from_str(VirtualNode::ZERO, "cc"), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// } +// ) +// .await +// .unwrap() +// .unwrap() +// ); +// // check iter +// let iter = hummock_storage +// .iter( +// ( +// Bound::Included(gen_key_from_str(VirtualNode::ZERO, "aa")), +// Bound::Included(gen_key_from_str(VirtualNode::ZERO, "cc")), +// ), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// }, +// ) +// .await +// .unwrap(); +// futures::pin_mut!(iter); +// assert_eq!( +// ( +// FullKey::new( +// TableId::default(), +// gen_key_from_str(VirtualNode::ZERO, "aa"), +// epoch +// ), +// Bytes::from("111") +// ), +// iter.try_next().await.unwrap().unwrap() +// ); +// assert_eq!( +// ( +// FullKey::new( +// TableId::default(), +// gen_key_from_str(VirtualNode::ZERO, "bb"), +// epoch +// ), +// Bytes::from("222") +// ), +// iter.try_next().await.unwrap().unwrap() +// ); +// assert_eq!( +// ( +// FullKey::new( +// TableId::default(), +// gen_key_from_str(VirtualNode::ZERO, "cc"), +// epoch +// ), +// Bytes::from("333") +// ), +// iter.try_next().await.unwrap().unwrap() +// ); +// assert!(iter.try_next().await.unwrap().is_none()); +// } +// }; + +// let batch1 = vec![ +// ( +// gen_key_from_str(VirtualNode::ZERO, "aa"), +// StorageValue::new_put("111"), +// ), +// ( +// gen_key_from_str(VirtualNode::ZERO, "bb"), +// StorageValue::new_put("222"), +// ), +// ( +// gen_key_from_str(VirtualNode::ZERO, "cc"), +// StorageValue::new_put("333"), +// ), +// ]; + +// let mut local = hummock_storage.new_local(NewLocalOptions::default()).await; +// local.init_for_test(epoch1).await.unwrap(); + +// local +// .ingest_batch( +// batch1.clone(), +// WriteOptions { +// epoch: epoch1, +// table_id: Default::default(), +// }, +// ) +// .await +// .unwrap(); +// assert_old_value(epoch1).await; + +// let assert_new_value = |epoch| { +// let hummock_storage = &hummock_storage; +// async move { +// // check point get +// assert_eq!( +// "111_new".as_bytes(), +// hummock_storage +// .get( +// gen_key_from_str(VirtualNode::ZERO, "aa"), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// } +// ) +// .await +// .unwrap() +// .unwrap() +// ); + +// assert!(hummock_storage +// .get( +// gen_key_from_str(VirtualNode::ZERO, "bb"), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// } +// ) +// .await +// .unwrap() +// .is_none()); +// assert_eq!( +// "333".as_bytes(), +// hummock_storage +// .get( +// gen_key_from_str(VirtualNode::ZERO, "cc"), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// } +// ) +// .await +// .unwrap() +// .unwrap() +// ); +// let iter = hummock_storage +// .iter( +// ( +// Bound::Included(gen_key_from_str(VirtualNode::ZERO, "aa")), +// Bound::Included(gen_key_from_str(VirtualNode::ZERO, "cc")), +// ), +// epoch, +// ReadOptions { +// cache_policy: CachePolicy::Fill(CachePriority::High), +// ..Default::default() +// }, +// ) +// .await +// .unwrap(); +// futures::pin_mut!(iter); +// assert_eq!( +// ( +// FullKey::new( +// TableId::default(), +// gen_key_from_str(VirtualNode::ZERO, "aa"), +// epoch +// ), +// Bytes::from("111_new") +// ), +// iter.try_next().await.unwrap().unwrap() +// ); +// assert_eq!( +// ( +// FullKey::new( +// TableId::default(), +// gen_key_from_str(VirtualNode::ZERO, "cc"), +// epoch +// ), +// Bytes::from("333") +// ), +// iter.try_next().await.unwrap().unwrap() +// ); +// assert!(iter.try_next().await.unwrap().is_none()); +// } +// }; + +// // Update aa, delete bb, cc unchanged +// let batch2 = vec![ +// ( +// gen_key_from_str(VirtualNode::ZERO, "aa"), +// StorageValue::new_put("111_new"), +// ), +// ( +// gen_key_from_str(VirtualNode::ZERO, "bb"), +// StorageValue::new_delete(), +// ), +// ]; + +// local +// .ingest_batch( +// batch2, +// WriteOptions { +// epoch: epoch1, +// table_id: Default::default(), +// }, +// ) +// .await +// .unwrap(); + +// assert_new_value(epoch1).await; + +// let epoch2 = epoch1 + 1; +// local.seal_current_epoch(epoch2, SealCurrentEpochOptions::for_test()); + +// // Write to epoch2 +// local +// .ingest_batch( +// batch1, +// WriteOptions { +// epoch: epoch2, +// table_id: Default::default(), +// }, +// ) +// .await +// .unwrap(); +// local.seal_current_epoch(u64::MAX, SealCurrentEpochOptions::for_test()); +// // Assert epoch 1 unchanged +// assert_new_value(epoch1).await; +// // Assert epoch 2 correctness +// assert_old_value(epoch2).await; + +// let ssts1 = hummock_storage +// .seal_and_sync_epoch(epoch1) +// .await +// .unwrap() +// .uncommitted_ssts; +// assert_new_value(epoch1).await; +// assert_old_value(epoch2).await; + +// let ssts2 = hummock_storage +// .seal_and_sync_epoch(epoch2) +// .await +// .unwrap() +// .uncommitted_ssts; +// assert_new_value(epoch1).await; +// assert_old_value(epoch2).await; + +// assert!(!ssts1.is_empty()); +// assert!(!ssts2.is_empty()); +// } #[tokio::test] async fn test_delete_get_v2() { @@ -1048,7 +1034,6 @@ async fn test_delete_get_inner( local .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: Default::default(), @@ -1072,7 +1057,6 @@ async fn test_delete_get_inner( local .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch2, table_id: Default::default(), @@ -1133,7 +1117,6 @@ async fn test_multiple_epoch_sync_inner( local .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: Default::default(), @@ -1151,7 +1134,6 @@ async fn test_multiple_epoch_sync_inner( local .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch2, table_id: Default::default(), @@ -1175,7 +1157,6 @@ async fn test_multiple_epoch_sync_inner( local .ingest_batch( batch3, - vec![], WriteOptions { epoch: epoch3, table_id: Default::default(), @@ -1285,7 +1266,7 @@ async fn test_gc_watermark_and_clear_shared_buffer() { None, ) .unwrap(); - local_hummock_storage.flush(Vec::new()).await.unwrap(); + local_hummock_storage.flush().await.unwrap(); assert_eq!( hummock_storage @@ -1302,7 +1283,7 @@ async fn test_gc_watermark_and_clear_shared_buffer() { Bytes::from("222"), ) .unwrap(); - local_hummock_storage.flush(Vec::new()).await.unwrap(); + local_hummock_storage.flush().await.unwrap(); assert_eq!( hummock_storage @@ -1351,14 +1332,10 @@ async fn test_gc_watermark_and_clear_shared_buffer() { min_object_id_epoch2, ); - hummock_storage.clear_shared_buffer().await.unwrap(); + drop(local_hummock_storage); - let read_version = local_hummock_storage.read_version(); + hummock_storage.clear_shared_buffer(epoch1).await; - let read_version = read_version.read(); - assert!(read_version.staging().imm.is_empty()); - assert!(read_version.staging().sst.is_empty()); - assert_eq!(read_version.committed().max_committed_epoch(), epoch1); assert_eq!( hummock_storage .sstable_object_id_manager() @@ -1419,7 +1396,6 @@ async fn test_replicated_local_hummock_storage() { local_hummock_storage .ingest_batch( batch1, - vec![], WriteOptions { epoch: epoch1, table_id: TEST_TABLE_ID, @@ -1446,13 +1422,13 @@ async fn test_replicated_local_hummock_storage() { [ Ok( ( - FullKey { UserKey { 233, TableKey { 000061616161 } }, epoch: 65536, epoch_with_gap: 65536}, + FullKey { UserKey { 233, TableKey { 000061616161 } }, epoch: 65536, epoch_with_gap: 65536, spill_offset: 0}, b"1111", ), ), Ok( ( - FullKey { UserKey { 233, TableKey { 000062626262 } }, epoch: 65536, epoch_with_gap: 65536}, + FullKey { UserKey { 233, TableKey { 000062626262 } }, epoch: 65536, epoch_with_gap: 65536, spill_offset: 0}, b"2222", ), ), @@ -1485,7 +1461,6 @@ async fn test_replicated_local_hummock_storage() { local_hummock_storage_2 .ingest_batch( batch2, - vec![], WriteOptions { epoch: epoch2, table_id: TEST_TABLE_ID, @@ -1507,13 +1482,13 @@ async fn test_replicated_local_hummock_storage() { [ Ok( ( - FullKey { UserKey { 233, TableKey { 000063636363 } }, epoch: 131072, epoch_with_gap: 131072}, + FullKey { UserKey { 233, TableKey { 000063636363 } }, epoch: 131072, epoch_with_gap: 131072, spill_offset: 0}, b"3333", ), ), Ok( ( - FullKey { UserKey { 233, TableKey { 000064646464 } }, epoch: 131072, epoch_with_gap: 131072}, + FullKey { UserKey { 233, TableKey { 000064646464 } }, epoch: 131072, epoch_with_gap: 131072, spill_offset: 0}, b"4444", ), ), diff --git a/src/storage/hummock_test/src/sync_point_tests.rs b/src/storage/hummock_test/src/sync_point_tests.rs index 7560507244fe5..313bb48c60afd 100644 --- a/src/storage/hummock_test/src/sync_point_tests.rs +++ b/src/storage/hummock_test/src/sync_point_tests.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::ops::Bound; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; use std::time::Duration; @@ -202,7 +201,7 @@ pub async fn compact_once( compact_task.compaction_filter_mask = compaction_filter_flag.bits(); // 3. compact let (_tx, rx) = tokio::sync::oneshot::channel(); - let (result_task, task_stats) = compact( + let ((result_task, task_stats), _) = compact( compact_ctx, compact_task.clone(), rx, @@ -222,6 +221,7 @@ pub async fn compact_once( .unwrap(); } +#[ignore] #[tokio::test] #[cfg(feature = "sync_point")] #[serial] @@ -297,7 +297,7 @@ async fn test_syncpoints_get_in_delete_range_boundary() { None, ) .unwrap(); - local.flush(Vec::new()).await.unwrap(); + local.flush().await.unwrap(); local.seal_current_epoch( test_epoch(101), risingwave_storage::store::SealCurrentEpochOptions::for_test(), @@ -325,13 +325,14 @@ async fn test_syncpoints_get_in_delete_range_boundary() { None, ) .unwrap(); - local - .flush(vec![( - Bound::Included(Bytes::from(b"\0\0ggg".as_slice())), - Bound::Excluded(Bytes::from(b"\0\0hhh".as_slice())), - )]) - .await - .unwrap(); + // local + // .flush(vec![( + // Bound::Included(Bytes::from(b"\0\0ggg".as_slice())), + // Bound::Excluded(Bytes::from(b"\0\0hhh".as_slice())), + // )]) + // .await + // .unwrap(); + local.flush().await.unwrap(); local.seal_current_epoch( test_epoch(102), risingwave_storage::store::SealCurrentEpochOptions::for_test(), @@ -359,13 +360,14 @@ async fn test_syncpoints_get_in_delete_range_boundary() { None, ) .unwrap(); - local - .flush(vec![( - Bound::Included(Bytes::from(b"\0\0jjj".as_slice())), - Bound::Excluded(Bytes::from(b"\0\0kkk".as_slice())), - )]) - .await - .unwrap(); + // local + // .flush(vec![( + // Bound::Included(Bytes::from(b"\0\0jjj".as_slice())), + // Bound::Excluded(Bytes::from(b"\0\0kkk".as_slice())), + // )]) + // .await + // .unwrap(); + local.flush().await.unwrap(); local.seal_current_epoch( test_epoch(103), risingwave_storage::store::SealCurrentEpochOptions::for_test(), @@ -394,7 +396,7 @@ async fn test_syncpoints_get_in_delete_range_boundary() { None, ) .unwrap(); - local.flush(Vec::new()).await.unwrap(); + local.flush().await.unwrap(); local.seal_current_epoch( u64::MAX, risingwave_storage::store::SealCurrentEpochOptions::for_test(), diff --git a/src/storage/hummock_test/src/test_utils.rs b/src/storage/hummock_test/src/test_utils.rs index 8410a6bbc46e0..f5d1a10a18839 100644 --- a/src/storage/hummock_test/src/test_utils.rs +++ b/src/storage/hummock_test/src/test_utils.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::ops::Bound; use std::sync::Arc; use bytes::Bytes; @@ -35,7 +34,7 @@ use risingwave_storage::filter_key_extractor::{ RpcFilterKeyExtractorManager, }; use risingwave_storage::hummock::backup_reader::BackupReader; -use risingwave_storage::hummock::event_handler::{HummockEvent, HummockVersionUpdate}; +use risingwave_storage::hummock::event_handler::HummockVersionUpdate; use risingwave_storage::hummock::iterator::test_utils::mock_sstable_store; use risingwave_storage::hummock::local_version::pinned_version::PinnedVersion; use risingwave_storage::hummock::observer_manager::HummockObserverNode; @@ -54,8 +53,8 @@ pub async fn prepare_first_valid_version( worker_node: WorkerNode, ) -> ( PinnedVersion, - UnboundedSender, - UnboundedReceiver, + UnboundedSender, + UnboundedReceiver, ) { let (tx, mut rx) = unbounded_channel(); let notification_client = @@ -74,7 +73,7 @@ pub async fn prepare_first_valid_version( .await; observer_manager.start().await; let hummock_version = match rx.recv().await { - Some(HummockEvent::VersionUpdate(HummockVersionUpdate::PinnedVersion(version))) => version, + Some(HummockVersionUpdate::PinnedVersion(version)) => version, _ => unreachable!("should be full version"), }; @@ -90,7 +89,6 @@ pub trait TestIngestBatch: LocalStateStore { async fn ingest_batch( &mut self, kv_pairs: Vec<(TableKey, StorageValue)>, - delete_ranges: Vec<(Bound, Bound)>, write_options: WriteOptions, ) -> StorageResult; } @@ -100,7 +98,6 @@ impl TestIngestBatch for S { async fn ingest_batch( &mut self, kv_pairs: Vec<(TableKey, StorageValue)>, - delete_ranges: Vec<(Bound, Bound)>, write_options: WriteOptions, ) -> StorageResult { assert_eq!(self.epoch(), write_options.epoch); @@ -110,10 +107,11 @@ impl TestIngestBatch for S { Some(value) => self.insert(key, value, None)?, } } - self.flush(delete_ranges).await + self.flush().await } } +#[cfg(test)] #[async_trait::async_trait] pub(crate) trait HummockStateStoreTestTrait: StateStore { fn get_pinned_version(&self) -> PinnedVersion; @@ -123,6 +121,7 @@ pub(crate) trait HummockStateStoreTestTrait: StateStore { } } +#[cfg(test)] impl HummockStateStoreTestTrait for HummockStorage { fn get_pinned_version(&self) -> PinnedVersion { self.get_pinned_version() diff --git a/src/storage/hummock_trace/src/collector.rs b/src/storage/hummock_trace/src/collector.rs index 751bcfd001a6b..b1a269a4620ee 100644 --- a/src/storage/hummock_trace/src/collector.rs +++ b/src/storage/hummock_trace/src/collector.rs @@ -214,8 +214,11 @@ impl TraceSpan { Self::new_global_op(Operation::SealCurrentEpoch { epoch, opts }, storage_type) } - pub fn new_clear_shared_buffer_span() -> MayTraceSpan { - Self::new_global_op(Operation::ClearSharedBuffer, StorageType::Global) + pub fn new_clear_shared_buffer_span(prev_epoch: u64) -> MayTraceSpan { + Self::new_global_op( + Operation::ClearSharedBuffer(prev_epoch), + StorageType::Global, + ) } pub fn new_validate_read_epoch_span(epoch: HummockReadEpoch) -> MayTraceSpan { @@ -310,15 +313,8 @@ impl TraceSpan { Self::new_global_op(Operation::DropLocalStorage, storage_type) } - pub fn new_flush_span( - delete_range: Vec<(Bound, Bound)>, - storage_type: StorageType, - ) -> MayTraceSpan { - let delete_range = delete_range - .into_iter() - .map(|(k, v)| (k.map(Bytes::into), v.map(Bytes::into))) - .collect(); - Self::new_global_op(Operation::Flush(delete_range), storage_type) + pub fn new_flush_span(storage_type: StorageType) -> MayTraceSpan { + Self::new_global_op(Operation::Flush, storage_type) } pub fn new_try_flush_span(storage_type: StorageType) -> MayTraceSpan { diff --git a/src/storage/hummock_trace/src/error.rs b/src/storage/hummock_trace/src/error.rs index f160a103325d9..e80a18aacc869 100644 --- a/src/storage/hummock_trace/src/error.rs +++ b/src/storage/hummock_trace/src/error.rs @@ -62,4 +62,7 @@ pub enum TraceError { #[error("failed to flush")] FlushFailed, + + #[error("failed to try_flush")] + TryFlushFailed, } diff --git a/src/storage/hummock_trace/src/opts.rs b/src/storage/hummock_trace/src/opts.rs index 54a1ca0340ec6..f5b749c83fb51 100644 --- a/src/storage/hummock_trace/src/opts.rs +++ b/src/storage/hummock_trace/src/opts.rs @@ -13,10 +13,12 @@ // limitations under the License. use bincode::{Decode, Encode}; +use risingwave_common::buffer::Bitmap; use risingwave_common::cache::CachePriority; use risingwave_common::catalog::{TableId, TableOption}; use risingwave_common::util::epoch::EpochPair; use risingwave_hummock_sdk::HummockReadEpoch; +use risingwave_pb::common::PbBuffer; use crate::TracedBytes; @@ -58,7 +60,7 @@ impl From for CachePriority { } } -#[derive(Copy, Encode, Decode, PartialEq, Eq, Debug, Clone, Hash)] +#[derive(Encode, Decode, PartialEq, Eq, Debug, Clone)] pub struct TracedTableId { pub table_id: u32, } @@ -114,7 +116,7 @@ pub struct TracedWriteOptions { pub table_id: TracedTableId, } -#[derive(Encode, Decode, PartialEq, Eq, Debug, Clone, Copy, Hash)] +#[derive(Encode, Decode, PartialEq, Eq, Debug, Clone)] pub struct TracedTableOption { pub retention_seconds: Option, } @@ -135,13 +137,13 @@ impl From for TableOption { } } -#[derive(Encode, Decode, PartialEq, Eq, Debug, Clone, Copy, Hash)] +#[derive(Encode, Decode, PartialEq, Eq, Debug, Clone)] pub enum TracedOpConsistencyLevel { Inconsistent, ConsistentOldValue, } -#[derive(Encode, Decode, PartialEq, Eq, Debug, Clone, Copy, Hash)] +#[derive(Encode, Decode, PartialEq, Eq, Debug, Clone)] pub struct TracedNewLocalOptions { pub table_id: TracedTableId, pub op_consistency_level: TracedOpConsistencyLevel, @@ -165,7 +167,7 @@ impl TracedNewLocalOptions { pub type TracedHummockEpoch = u64; -#[derive(Debug, Clone, Copy, PartialEq, Eq, Decode, Encode)] +#[derive(Debug, Clone, PartialEq, Eq, Decode, Encode)] pub enum TracedHummockReadEpoch { Committed(TracedHummockEpoch), Current(TracedHummockEpoch), @@ -195,7 +197,7 @@ impl From for HummockReadEpoch { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Decode, Encode)] +#[derive(Debug, Clone, PartialEq, Eq, Decode, Encode)] pub struct TracedEpochPair { pub curr: TracedHummockEpoch, pub prev: TracedHummockEpoch, @@ -219,13 +221,41 @@ impl From for EpochPair { } } -#[derive(Debug, Clone, Copy, PartialEq, Eq, Decode, Encode)] +#[derive(Debug, Clone, PartialEq, Eq, Decode, Encode)] pub struct TracedInitOptions { pub epoch: TracedEpochPair, + pub vnodes: TracedBitmap, } #[derive(Debug, Clone, PartialEq, Eq, Decode, Encode)] pub struct TracedSealCurrentEpochOptions { // The watermark is serialized into protobuf pub table_watermarks: Option<(bool, Vec>)>, + pub switch_op_consistency_level: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq, Decode, Encode)] +pub struct TracedBitmap { + pub compression: i32, + pub body: Vec, +} + +impl From for TracedBitmap { + fn from(value: Bitmap) -> Self { + let pb = value.to_protobuf(); + Self { + compression: pb.compression, + body: pb.body, + } + } +} + +impl From for Bitmap { + fn from(value: TracedBitmap) -> Self { + let pb = PbBuffer { + compression: value.compression, + body: value.body, + }; + Bitmap::from(&pb) + } } diff --git a/src/storage/hummock_trace/src/record.rs b/src/storage/hummock_trace/src/record.rs index cd00dcb0a4411..f8e4ceed65449 100644 --- a/src/storage/hummock_trace/src/record.rs +++ b/src/storage/hummock_trace/src/record.rs @@ -170,7 +170,7 @@ pub enum Operation { TryWaitEpoch(TracedHummockReadEpoch), /// clear shared buffer - ClearSharedBuffer, + ClearSharedBuffer(u64), /// Seal current epoch SealCurrentEpoch { @@ -187,7 +187,7 @@ pub enum Operation { TryFlush, - Flush(Vec<(Bound, Bound)>), + Flush, /// Finish operation of Hummock. Finish, } @@ -299,7 +299,6 @@ pub enum OperationResult { Sync(TraceResult), NotifyHummock(TraceResult<()>), TryWaitEpoch(TraceResult<()>), - ClearSharedBuffer(TraceResult<()>), ValidateReadEpoch(TraceResult<()>), LocalStorageEpoch(TraceResult), LocalStorageIsDirty(TraceResult), diff --git a/src/storage/hummock_trace/src/replay/mod.rs b/src/storage/hummock_trace/src/replay/mod.rs index c411a8b64d11a..046ab67b18607 100644 --- a/src/storage/hummock_trace/src/replay/mod.rs +++ b/src/storage/hummock_trace/src/replay/mod.rs @@ -63,10 +63,8 @@ pub trait LocalReplay: LocalReplayRead + ReplayWrite + Send + Sync { fn seal_current_epoch(&mut self, next_epoch: u64, opts: TracedSealCurrentEpochOptions); fn is_dirty(&self) -> bool; fn epoch(&self) -> u64; - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> Result; + async fn try_flush(&mut self) -> Result<()>; + async fn flush(&mut self) -> Result; } pub trait GlobalReplay: ReplayRead + ReplayStateStore + Send + Sync {} @@ -122,7 +120,7 @@ pub trait ReplayStateStore { async fn notify_hummock(&self, info: Info, op: RespOperation, version: u64) -> Result; async fn new_local(&self, opts: TracedNewLocalOptions) -> Box; async fn try_wait_epoch(&self, epoch: HummockReadEpoch) -> Result<()>; - async fn clear_shared_buffer(&self) -> Result<()>; + async fn clear_shared_buffer(&self, prev_epoch: u64); fn validate_read_epoch(&self, epoch: HummockReadEpoch) -> Result<()>; } @@ -154,7 +152,7 @@ mock! { ) -> Result; async fn new_local(&self, opts: TracedNewLocalOptions) -> Box; async fn try_wait_epoch(&self, epoch: HummockReadEpoch) -> Result<()>; - async fn clear_shared_buffer(&self) -> Result<()>; + async fn clear_shared_buffer(&self, prev_epoch: u64); fn validate_read_epoch(&self, epoch: HummockReadEpoch) -> Result<()>; } impl GlobalReplay for GlobalReplayInterface{} @@ -188,7 +186,8 @@ mock! { fn seal_current_epoch(&mut self, next_epoch: u64, opts: TracedSealCurrentEpochOptions); fn is_dirty(&self) -> bool; fn epoch(&self) -> u64; - async fn flush(&mut self, delete_ranges: Vec<(Bound, Bound)>) -> Result; + async fn flush(&mut self) -> Result; + async fn try_flush(&mut self) -> Result<()>; } } diff --git a/src/storage/hummock_trace/src/replay/runner.rs b/src/storage/hummock_trace/src/replay/runner.rs index 02bfd0277662f..2683b0ddcecc1 100644 --- a/src/storage/hummock_trace/src/replay/runner.rs +++ b/src/storage/hummock_trace/src/replay/runner.rs @@ -104,7 +104,7 @@ mod tests { let storage_type4 = StorageType::Global; let actor_1 = vec![ - (0, Operation::NewLocalStorage(opts1, 1)), + (0, Operation::NewLocalStorage(opts1.clone(), 1)), ( 1, Operation::get( @@ -135,7 +135,7 @@ mod tests { .map(|(record_id, op)| Ok(Record::new(storage_type1, record_id, op))); let actor_2 = vec![ - (4, Operation::NewLocalStorage(opts2, 2)), + (4, Operation::NewLocalStorage(opts2.clone(), 2)), ( 5, Operation::get( @@ -166,7 +166,7 @@ mod tests { .map(|(record_id, op)| Ok(Record::new(storage_type2, record_id, op))); let actor_3 = vec![ - (8, Operation::NewLocalStorage(opts3, 3)), + (8, Operation::NewLocalStorage(opts3.clone(), 3)), ( 9, Operation::get( diff --git a/src/storage/hummock_trace/src/replay/worker.rs b/src/storage/hummock_trace/src/replay/worker.rs index d68196d61a5e4..29e814c3f9ade 100644 --- a/src/storage/hummock_trace/src/replay/worker.rs +++ b/src/storage/hummock_trace/src/replay/worker.rs @@ -284,7 +284,7 @@ impl ReplayWorker { } Operation::NewLocalStorage(new_local_opts, id) => { assert_ne!(storage_type, StorageType::Global); - local_storage_opts_map.insert(id, new_local_opts); + local_storage_opts_map.insert(id, new_local_opts.clone()); let local_storage = replay.new_local(new_local_opts).await; local_storages.insert(storage_type, local_storage); } @@ -327,22 +327,9 @@ impl ReplayWorker { ); } } - Operation::ClearSharedBuffer => { + Operation::ClearSharedBuffer(prev_epoch) => { assert_eq!(storage_type, StorageType::Global); - let res = res_rx.recv().await.expect("recv result failed"); - if let OperationResult::ClearSharedBuffer(expected) = res { - let actual = replay.clear_shared_buffer().await; - assert_eq!( - TraceResult::from(actual), - expected, - "clear_shared_buffer wrong" - ); - } else { - panic!( - "wrong clear_shared_buffer result, expect epoch result, but got {:?}", - res - ); - } + replay.clear_shared_buffer(prev_epoch).await; } Operation::SealCurrentEpoch { epoch, opts } => { assert_ne!(storage_type, StorageType::Global); @@ -399,12 +386,12 @@ impl ReplayWorker { ); } } - Operation::Flush(delete_range) => { + Operation::Flush => { assert_ne!(storage_type, StorageType::Global); let local_storage = local_storages.get_mut(&storage_type).unwrap(); let res = res_rx.recv().await.expect("recv result failed"); if let OperationResult::Flush(expected) = res { - let actual = local_storage.flush(delete_range).await; + let actual = local_storage.flush().await; assert_eq!(TraceResult::from(actual), expected, "flush wrong"); } else { panic!("wrong flush result, expect flush result, but got {:?}", res); @@ -414,9 +401,8 @@ impl ReplayWorker { assert_ne!(storage_type, StorageType::Global); let local_storage = local_storages.get_mut(&storage_type).unwrap(); let res = res_rx.recv().await.expect("recv result failed"); - let delete_range = vec![]; if let OperationResult::TryFlush(_) = res { - let _ = local_storage.flush(delete_range).await; + let _ = local_storage.try_flush().await; // todo(wcy-fdu): unify try_flush and flush interface, do not return usize. // assert_eq!(TraceResult::from(actual), expected, "try flush wrong"); } else { diff --git a/src/storage/hummock_trace/src/write.rs b/src/storage/hummock_trace/src/write.rs index 3038ffb9a7ee8..528cdac5f659d 100644 --- a/src/storage/hummock_trace/src/write.rs +++ b/src/storage/hummock_trace/src/write.rs @@ -30,13 +30,6 @@ pub(crate) static MAGIC_BYTES: MagicBytes = 0x484D5452; // HMTR pub(crate) trait TraceWriter { fn write(&mut self, record: Record) -> Result; fn flush(&mut self) -> Result<()>; - fn write_all(&mut self, records: Vec) -> Result { - let mut total_size = 0; - for r in records { - total_size += self.write(r)? - } - Ok(total_size) - } } /// Serializer serializes a record to std write. diff --git a/src/storage/src/error.rs b/src/storage/src/error.rs index dbdbf50e330fa..c76913ebe6eb9 100644 --- a/src/storage/src/error.rs +++ b/src/storage/src/error.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::error::{ErrorCode, RwError}; use risingwave_common::util::value_encoding::error::ValueEncodingError; use thiserror::Error; @@ -59,9 +58,3 @@ pub enum ErrorKind { } pub type StorageResult = std::result::Result; - -impl From for RwError { - fn from(s: StorageError) -> Self { - ErrorCode::StorageError(Box::new(s)).into() - } -} diff --git a/src/storage/src/filter_key_extractor.rs b/src/storage/src/filter_key_extractor.rs index ac3ef6eb02e8c..b9c03b36d6126 100644 --- a/src/storage/src/filter_key_extractor.rs +++ b/src/storage/src/filter_key_extractor.rs @@ -27,6 +27,7 @@ use risingwave_hummock_sdk::key::{get_table_id, TABLE_PREFIX_LEN}; use risingwave_pb::catalog::Table; use risingwave_rpc_client::error::{Result as RpcResult, RpcError}; use risingwave_rpc_client::MetaClient; +use thiserror_ext::AsReport; use crate::hummock::{HummockError, HummockResult}; @@ -315,8 +316,8 @@ impl FilterKeyExtractorManagerInner { .await .map_err(|e| { HummockError::other(format!( - "request rpc list_tables for meta failed because {:?}", - e + "request rpc list_tables for meta failed: {}", + e.as_report() )) })?; let mut guard = self.table_id_to_filter_key_extractor.write(); @@ -432,14 +433,13 @@ pub type FilterKeyExtractorManagerRef = Arc; #[cfg(test)] mod tests { - use std::collections::{HashMap, HashSet}; + use std::collections::HashSet; use std::mem; use std::sync::Arc; use bytes::{BufMut, BytesMut}; use itertools::Itertools; use risingwave_common::catalog::ColumnDesc; - use risingwave_common::constants::hummock::PROPERTIES_RETENTION_SECOND_KEY; use risingwave_common::hash::VirtualNode; use risingwave_common::row::OwnedRow; use risingwave_common::types::DataType; @@ -530,10 +530,7 @@ mod tests { optional_associated_source_id: None, append_only: false, owner: risingwave_common::catalog::DEFAULT_SUPER_USER_ID, - properties: HashMap::from([( - String::from(PROPERTIES_RETENTION_SECOND_KEY), - String::from("300"), - )]), + retention_seconds: Some(300), fragment_id: 0, dml_fragment_id: None, initialized_at_epoch: None, diff --git a/src/storage/src/hummock/backup_reader.rs b/src/storage/src/hummock/backup_reader.rs index f7ca09eea8e4b..0b99970f26e75 100644 --- a/src/storage/src/hummock/backup_reader.rs +++ b/src/storage/src/hummock/backup_reader.rs @@ -27,8 +27,10 @@ use risingwave_backup::storage::{MetaSnapshotStorage, ObjectStoreMetaSnapshotSto use risingwave_backup::{meta_snapshot_v1, MetaSnapshotId}; use risingwave_common::config::ObjectStoreConfig; use risingwave_common::system_param::local_manager::SystemParamsReaderRef; +use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_object_store::object::build_remote_object_store; use risingwave_object_store::object::object_metrics::ObjectStoreMetrics; +use thiserror_ext::AsReport; use crate::error::{StorageError, StorageResult}; use crate::hummock::local_version::pinned_version::{PinVersionAction, PinnedVersion}; @@ -130,7 +132,7 @@ impl BackupReader { } if let Err(e) = current_store.0.refresh_manifest().await { // reschedule refresh request - tracing::warn!("failed to refresh backup manifest, will retry. {}", e); + tracing::warn!(error = %e.as_report(), "failed to refresh backup manifest, will retry"); let backup_reader_clone = backup_reader.clone(); tokio::spawn(async move { tokio::time::sleep(Duration::from_secs(60)).await; @@ -154,10 +156,9 @@ impl BackupReader { } pub fn try_refresh_manifest(self: &BackupReaderRef, min_manifest_id: u64) { - let _ = self - .refresh_tx - .send(min_manifest_id) - .inspect_err(|e| tracing::warn!("failed to send refresh_manifest request {}", e)); + let _ = self.refresh_tx.send(min_manifest_id).inspect_err(|_| { + tracing::warn!(min_manifest_id, "failed to send refresh_manifest request") + }); } /// Tries to get a hummock version eligible for querying `epoch`. @@ -197,7 +198,11 @@ impl BackupReader { // TODO: change to v2 let snapshot: meta_snapshot_v1::MetaSnapshotV1 = current_store.0.get(snapshot_id).await.map_err(|e| { - format!("failed to get meta snapshot {}. {}", snapshot_id, e) + format!( + "failed to get meta snapshot {}: {}", + snapshot_id, + e.as_report() + ) })?; let version_holder = build_version_holder(snapshot); let version_clone = version_holder.0.clone(); @@ -237,10 +242,9 @@ impl BackupReader { if let Err(e) = self.set_store(config.clone()).await { // Retry is driven by periodic system params notification. tracing::warn!( - "failed to update backup reader: url={}, dir={}, {:#?}", - config.0, - config.1, - e + url = config.0, dir = config.1, + error = %e.as_report(), + "failed to update backup reader", ); } } diff --git a/src/storage/src/hummock/block_cache.rs b/src/storage/src/hummock/block_cache.rs index 5df791e21cf0f..9023f045c0557 100644 --- a/src/storage/src/hummock/block_cache.rs +++ b/src/storage/src/hummock/block_cache.rs @@ -34,9 +34,9 @@ const MIN_BUFFER_SIZE_PER_SHARD: usize = 256 * 1024 * 1024; type CachedBlockEntry = CacheableEntry<(HummockSstableObjectId, u64), Box>; enum BlockEntry { - Cache(CachedBlockEntry), - Owned(Box), - RefEntry(Arc), + Cache(#[allow(dead_code)] CachedBlockEntry), + Owned(#[allow(dead_code)] Box), + RefEntry(#[allow(dead_code)] Arc), } pub struct BlockHolder { @@ -62,7 +62,7 @@ impl BlockHolder { } pub fn from_cached_block(entry: CachedBlockEntry) -> Self { - let ptr = entry.value().as_ref() as *const _; + let ptr = entry.as_ref() as *const _; Self { _handle: BlockEntry::Cache(entry), block: ptr, diff --git a/src/storage/src/hummock/compactor/compaction_filter.rs b/src/storage/src/hummock/compactor/compaction_filter.rs index e67460129b96d..14927b5865b38 100644 --- a/src/storage/src/hummock/compactor/compaction_filter.rs +++ b/src/storage/src/hummock/compactor/compaction_filter.rs @@ -15,7 +15,6 @@ use std::collections::{HashMap, HashSet}; use dyn_clone::DynClone; -use risingwave_common::catalog::hummock::TABLE_OPTION_DUMMY_RETENTION_SECOND; use risingwave_hummock_sdk::key::FullKey; pub trait CompactionFilter: Send + Sync + DynClone { @@ -80,7 +79,7 @@ impl CompactionFilter for TtlCompactionFilter { } match self.table_id_to_ttl.get(&table_id) { Some(ttl_second_u32) => { - assert!(*ttl_second_u32 != TABLE_OPTION_DUMMY_RETENTION_SECOND); + assert!(*ttl_second_u32 > 0); // default to zero. let ttl_mill = *ttl_second_u32 as u64 * 1000; let min_epoch = Epoch(self.expire_epoch).subtract_ms(ttl_mill); diff --git a/src/storage/src/hummock/compactor/compaction_utils.rs b/src/storage/src/hummock/compactor/compaction_utils.rs index c38ed4b65a82a..758a6d1253451 100644 --- a/src/storage/src/hummock/compactor/compaction_utils.rs +++ b/src/storage/src/hummock/compactor/compaction_utils.rs @@ -20,6 +20,7 @@ use std::sync::Arc; use itertools::Itertools; use risingwave_common::constants::hummock::CompactionFilterFlag; +use risingwave_hummock_sdk::compaction_group::StateTableId; use risingwave_hummock_sdk::key::FullKey; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; @@ -37,13 +38,14 @@ use crate::hummock::compactor::{ TtlCompactionFilter, }; use crate::hummock::iterator::{ - ForwardMergeRangeIterator, SkipWatermarkIterator, UnorderedMergeIteratorInner, UserIterator, + Forward, ForwardMergeRangeIterator, HummockIterator, MergeIterator, SkipWatermarkIterator, + UserIterator, }; use crate::hummock::multi_builder::TableBuilderFactory; use crate::hummock::sstable::DEFAULT_ENTRY_SIZE; use crate::hummock::{ CachePolicy, FilterBuilder, GetObjectId, HummockResult, MemoryLimiter, SstableBuilder, - SstableBuilderOptions, SstableWriterFactory, SstableWriterOptions, + SstableBuilderOptions, SstableDeleteRangeIterator, SstableWriterFactory, SstableWriterOptions, }; use crate::monitor::StoreLocalStatistic; @@ -132,7 +134,6 @@ pub struct TaskConfig { } pub fn build_multi_compaction_filter(compact_task: &CompactTask) -> MultiCompactionFilter { - use risingwave_common::catalog::TableOption; let mut multi_filter = MultiCompactionFilter::default(); let compaction_filter_flag = CompactionFilterFlag::from_bits(compact_task.compaction_filter_mask).unwrap_or_default(); @@ -148,11 +149,11 @@ pub fn build_multi_compaction_filter(compact_task: &CompactTask) -> MultiCompact let id_to_ttl = compact_task .table_options .iter() - .filter(|id_to_option| { - let table_option: TableOption = id_to_option.1.into(); - table_option.retention_seconds.is_some() + .filter_map(|(id, option)| { + option + .retention_seconds + .and_then(|ttl| if ttl > 0 { Some((*id, ttl)) } else { None }) }) - .map(|id_to_option| (*id_to_option.0, id_to_option.1.retention_seconds)) .collect(); let ttl_filter = Box::new(TtlCompactionFilter::new( @@ -240,7 +241,6 @@ pub async fn generate_splits( .sstable_store .sstable(sstable_info, &mut StoreLocalStatistic::default()) .await? - .value() .meta .block_metas .iter() @@ -312,11 +312,36 @@ pub fn estimate_task_output_capacity(context: CompactorContext, task: &CompactTa std::cmp::min(capacity, total_input_uncompressed_file_size as usize) } +/// Compare result of compaction task and input. The data saw by user shall not change after applying compaction result. pub async fn check_compaction_result( compact_task: &CompactTask, context: CompactorContext, -) -> HummockResult<()> { +) -> HummockResult { + let has_ttl = compact_task + .table_options + .iter() + .any(|(_, table_option)| table_option.retention_seconds.is_some_and(|ttl| ttl > 0)); + + let mut compact_table_ids = compact_task + .input_ssts + .iter() + .flat_map(|level| level.table_infos.iter()) + .flat_map(|sst| sst.table_ids.clone()) + .collect_vec(); + compact_table_ids.sort(); + compact_table_ids.dedup(); + let existing_table_ids: HashSet = + HashSet::from_iter(compact_task.existing_table_ids.clone()); + let need_clean_state_table = compact_table_ids + .iter() + .any(|table_id| !existing_table_ids.contains(table_id)); + // This check method does not consider dropped keys by compaction filter. + if has_ttl || need_clean_state_table { + return Ok(true); + } + let mut table_iters = Vec::new(); + let mut del_iter = ForwardMergeRangeIterator::default(); let compact_io_retry_time = context.storage_opts.compact_iter_recreate_timeout_ms; for level in &compact_task.input_ssts { if level.table_infos.is_empty() { @@ -326,6 +351,7 @@ pub async fn check_compaction_result( // Do not need to filter the table because manager has done it. if level.level_type == LevelType::Nonoverlapping as i32 { debug_assert!(can_concat(&level.table_infos)); + del_iter.add_concat_iter(level.table_infos.clone(), context.sstable_store.clone()); table_iters.push(ConcatSstableIterator::new( compact_task.existing_table_ids.clone(), @@ -336,8 +362,13 @@ pub async fn check_compaction_result( compact_io_retry_time, )); } else { + let mut stats = StoreLocalStatistic::default(); for table_info in &level.table_infos { - assert_eq!(table_info.range_tombstone_count, 0); + let table = context + .sstable_store + .sstable(table_info, &mut stats) + .await?; + del_iter.add_sst_iter(SstableDeleteRangeIterator::new(table)); table_iters.push(ConcatSstableIterator::new( compact_task.existing_table_ids.clone(), vec![table_info.clone()], @@ -349,14 +380,20 @@ pub async fn check_compaction_result( } } } - let iter = UnorderedMergeIteratorInner::for_compactor(table_iters); - let mut left_iter = UserIterator::new( + + let iter = MergeIterator::for_compactor(table_iters); + let left_iter = UserIterator::new( SkipWatermarkIterator::from_safe_epoch_watermarks(iter, &compact_task.table_watermarks), (Bound::Unbounded, Bound::Unbounded), u64::MAX, 0, None, - ForwardMergeRangeIterator::default(), + del_iter, + ); + let mut del_iter = ForwardMergeRangeIterator::default(); + del_iter.add_concat_iter( + compact_task.sorted_output_ssts.clone(), + context.sstable_store.clone(), ); let iter = ConcatSstableIterator::new( compact_task.existing_table_ids.clone(), @@ -366,22 +403,94 @@ pub async fn check_compaction_result( Arc::new(TaskProgress::default()), compact_io_retry_time, ); - let mut right_iter = UserIterator::new( + let right_iter = UserIterator::new( SkipWatermarkIterator::from_safe_epoch_watermarks(iter, &compact_task.table_watermarks), (Bound::Unbounded, Bound::Unbounded), u64::MAX, 0, None, - ForwardMergeRangeIterator::default(), + del_iter, + ); + + check_result(left_iter, right_iter).await +} + +pub async fn check_flush_result>( + left_iter: UserIterator, + existing_table_ids: Vec, + sort_ssts: Vec, + context: CompactorContext, +) -> HummockResult { + let mut del_iter = ForwardMergeRangeIterator::default(); + del_iter.add_concat_iter(sort_ssts.clone(), context.sstable_store.clone()); + let iter = ConcatSstableIterator::new( + existing_table_ids.clone(), + sort_ssts.clone(), + KeyRange::inf(), + context.sstable_store.clone(), + Arc::new(TaskProgress::default()), + 0, + ); + let right_iter = UserIterator::new( + iter, + (Bound::Unbounded, Bound::Unbounded), + u64::MAX, + 0, + None, + del_iter, ); + check_result(left_iter, right_iter).await +} + +async fn check_result< + I1: HummockIterator, + I2: HummockIterator, +>( + mut left_iter: UserIterator, + mut right_iter: UserIterator, +) -> HummockResult { left_iter.rewind().await?; right_iter.rewind().await?; + let mut right_count = 0; + let mut left_count = 0; while left_iter.is_valid() && right_iter.is_valid() { - assert_eq!(left_iter.key(), right_iter.key()); - assert_eq!(left_iter.value(), right_iter.value()); + if left_iter.key() != right_iter.key() { + tracing::error!( + "The key of input and output not equal. key: {:?} vs {:?}", + left_iter.key(), + right_iter.key() + ); + return Ok(false); + } + if left_iter.value() != right_iter.value() { + tracing::error!( + "The value of input and output not equal. key: {:?}, value: {:?} vs {:?}", + left_iter.key(), + left_iter.value(), + right_iter.value() + ); + return Ok(false); + } + left_iter.next().await?; + right_iter.next().await?; + left_count += 1; + right_count += 1; + } + while left_iter.is_valid() { + left_count += 1; left_iter.next().await?; + } + while right_iter.is_valid() { + right_count += 1; right_iter.next().await?; } - assert!(!left_iter.is_valid() && !right_iter.is_valid()); - Ok(()) + if left_count != right_count { + tracing::error!( + "The key count of input and output not equal: {} vs {}", + left_count, + right_count + ); + return Ok(false); + } + Ok(true) } diff --git a/src/storage/src/hummock/compactor/compactor_runner.rs b/src/storage/src/hummock/compactor/compactor_runner.rs index 0ab64f74a0fa3..50bfd2ae46baa 100644 --- a/src/storage/src/hummock/compactor/compactor_runner.rs +++ b/src/storage/src/hummock/compactor/compactor_runner.rs @@ -24,17 +24,18 @@ use risingwave_common::util::epoch::is_max_epoch; use risingwave_hummock_sdk::compact::{ compact_task_to_string, estimate_memory_for_compact_task, statistics_compact_task, }; -use risingwave_hummock_sdk::key::{FullKey, PointRange}; +use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker, PointRange}; use risingwave_hummock_sdk::key_range::{KeyRange, KeyRangeCommon}; use risingwave_hummock_sdk::table_stats::{add_table_stats_map, TableStats, TableStatsMap}; use risingwave_hummock_sdk::{can_concat, EpochWithGap, HummockEpoch}; use risingwave_pb::hummock::compact_task::{TaskStatus, TaskType}; use risingwave_pb::hummock::{BloomFilterType, CompactTask, LevelType}; +use thiserror_ext::AsReport; use tokio::sync::oneshot::Receiver; use super::iterator::MonitoredCompactorIterator; use super::task_progress::TaskProgress; -use super::{check_compaction_result, CompactionStatistics, TaskConfig}; +use super::{CompactionStatistics, TaskConfig}; use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorManager}; use crate::hummock::compactor::compaction_utils::{ build_multi_compaction_filter, estimate_task_output_capacity, generate_splits, @@ -45,10 +46,10 @@ use crate::hummock::compactor::{ fast_compactor_runner, CompactOutput, CompactionFilter, Compactor, CompactorContext, }; use crate::hummock::iterator::{ - Forward, ForwardMergeRangeIterator, HummockIterator, SkipWatermarkIterator, - UnorderedMergeIteratorInner, + Forward, ForwardMergeRangeIterator, HummockIterator, MergeIterator, SkipWatermarkIterator, }; use crate::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory}; +use crate::hummock::utils::MemoryTracker; use crate::hummock::value::HummockValue; use crate::hummock::{ BlockedXor16FilterBuilder, CachePolicy, CompactionDeleteRangeIterator, CompressionAlgorithm, @@ -233,7 +234,7 @@ impl CompactorRunner { Ok(( SkipWatermarkIterator::from_safe_epoch_watermarks( MonitoredCompactorIterator::new( - UnorderedMergeIteratorInner::for_compactor(table_iters), + MergeIterator::for_compactor(table_iters), task_progress.clone(), ), &self.compact_task.table_watermarks, @@ -251,7 +252,10 @@ pub async fn compact( mut shutdown_rx: Receiver<()>, object_id_getter: Box, filter_key_extractor_manager: FilterKeyExtractorManager, -) -> (CompactTask, HashMap) { +) -> ( + (CompactTask, HashMap), + Option, +) { let context = compactor_context.clone(); let group_label = compact_task.compaction_group_id.to_string(); let cur_level_label = compact_task.input_ssts[0].level_idx.to_string(); @@ -328,9 +332,12 @@ pub async fn compact( .await { Err(e) => { - tracing::error!("Failed to fetch filter key extractor tables [{:?}], it may caused by some RPC error {:?}", compact_task.existing_table_ids, e); + tracing::error!(error = %e.as_report(), "Failed to fetch filter key extractor tables [{:?}], it may caused by some RPC error", compact_task.existing_table_ids); let task_status = TaskStatus::ExecuteFailed; - return compact_done(compact_task, context.clone(), vec![], task_status); + return ( + compact_done(compact_task, context.clone(), vec![], task_status), + None, + ); } Ok(extractor) => extractor, }; @@ -344,7 +351,10 @@ pub async fn compact( if !removed_tables.is_empty() { tracing::error!("Failed to fetch filter key extractor tables [{:?}. [{:?}] may be removed by meta-service. ", compact_table_ids, removed_tables); let task_status = TaskStatus::ExecuteFailed; - return compact_done(compact_task, context.clone(), vec![], task_status); + return ( + compact_done(compact_task, context.clone(), vec![], task_status), + None, + ); } } @@ -357,7 +367,7 @@ pub async fn compact( let has_ttl = compact_task .table_options .iter() - .any(|(_, table_option)| table_option.retention_seconds > 0); + .any(|(_, table_option)| table_option.retention_seconds.is_some_and(|ttl| ttl > 0)); let mut task_status = TaskStatus::Success; // skip sst related to non-existent able_id to reduce io let sstable_infos = compact_task @@ -408,9 +418,12 @@ pub async fn compact( } } Err(e) => { - tracing::warn!("Failed to generate_splits {:#?}", e); + tracing::warn!(error = %e.as_report(), "Failed to generate_splits"); task_status = TaskStatus::ExecuteFailed; - return compact_done(compact_task, context.clone(), vec![], task_status); + return ( + compact_done(compact_task, context.clone(), vec![], task_status), + None, + ); } } } @@ -426,11 +439,14 @@ pub async fn compact( context.running_task_parallelism.load(Ordering::Relaxed), context.max_task_parallelism.load(Ordering::Relaxed), ); - return compact_done( - compact_task, - context.clone(), - vec![], - TaskStatus::NoAvailCpuResourceCanceled, + return ( + compact_done( + compact_task, + context.clone(), + vec![], + TaskStatus::NoAvailCpuResourceCanceled, + ), + None, ); } @@ -488,7 +504,10 @@ pub async fn compact( context.memory_limiter.quota() ); task_status = TaskStatus::NoAvailMemoryResourceCanceled; - return compact_done(compact_task, context.clone(), output_ssts, task_status); + return ( + compact_done(compact_task, context.clone(), output_ssts, task_status), + memory_detector, + ); } context.compactor_metrics.compact_task_pending_num.inc(); @@ -528,9 +547,9 @@ pub async fn compact( Err(e) => { task_status = TaskStatus::ExecuteFailed; tracing::warn!( - "Compaction task {} failed with error: {:#?}", + error = %e.as_report(), + "Compaction task {} failed with error", compact_task.task_id, - e ); } } @@ -546,17 +565,7 @@ pub async fn compact( cost_time, compact_task_to_string(&compact_task) ); - // TODO: remove this method after we have running risingwave cluster with fast compact algorithm stably for a long time. - if context.storage_opts.check_fast_compaction_result - && let Err(e) = check_compaction_result(&compact_task, context.clone()).await - { - tracing::error!( - "Failed to check fast compaction task {} because: {:?}", - compact_task.task_id, - e - ); - } - return (compact_task, table_stats); + return ((compact_task, table_stats), memory_detector); } for (split_index, _) in compact_task.splits.iter().enumerate() { let filter = multi_filter.clone(); @@ -608,18 +617,18 @@ pub async fn compact( Some(Ok(Err(e))) => { task_status = TaskStatus::ExecuteFailed; tracing::warn!( - "Compaction task {} failed with error: {:#?}", + error = %e.as_report(), + "Compaction task {} failed with error", compact_task.task_id, - e ); break; } Some(Err(e)) => { task_status = TaskStatus::JoinHandleFailed; tracing::warn!( - "Compaction task {} failed with join handle error: {:#?}", + error = %e.as_report(), + "Compaction task {} failed with join handle error", compact_task.task_id, - e ); break; } @@ -629,8 +638,6 @@ pub async fn compact( } } - drop(memory_detector); - if task_status != TaskStatus::Success { for abort_handle in abort_handles { abort_handle.abort(); @@ -651,12 +658,7 @@ pub async fn compact( cost_time, compact_task_to_string(&compact_task) ); - for level in &compact_task.input_ssts { - for table in &level.table_infos { - context.sstable_store.delete_cache(table.get_object_id()); - } - } - (compact_task, table_stats) + ((compact_task, table_stats), memory_detector) } /// Fills in the compact task and tries to report the task result to meta node. @@ -723,6 +725,10 @@ where if !task_config.gc_delete_keys && del_iter.is_valid() && !is_max_epoch(del_iter.earliest_epoch()) + && !compaction_filter.should_delete(FullKey::from_user_key( + full_key.user_key, + del_iter.earliest_epoch(), + )) { sst_builder .add_monotonic_delete(MonotonicDeleteEvent { @@ -743,7 +749,7 @@ where }; let max_key = end_key.to_ref(); - let mut last_key = FullKey::default(); + let mut full_key_tracker = FullKeyTracker::>::new(FullKey::default()); let mut watermark_can_see_last_key = false; let mut user_key_last_delete_epoch = HummockEpoch::MAX; let mut local_stats = StoreLocalStatistic::default(); @@ -757,18 +763,16 @@ where let mut iter_key = iter.key(); compaction_statistics.iter_total_key_counts += 1; - let mut is_new_user_key = - last_key.is_empty() || iter_key.user_key != last_key.user_key.as_ref(); - + let mut is_new_user_key = full_key_tracker.observe(iter.key()).is_some(); let mut drop = false; + // CRITICAL WARN: Because of memtable spill, there may be several versions of the same user-key share the same `pure_epoch`. Do not change this code unless necessary. let epoch = iter_key.epoch_with_gap.pure_epoch(); let value = iter.value(); if is_new_user_key { if !max_key.is_empty() && iter_key >= max_key { break; } - last_key.set(iter_key); watermark_can_see_last_key = false; user_key_last_delete_epoch = HummockEpoch::MAX; if value.is_delete() { @@ -779,23 +783,29 @@ where } if last_table_id.map_or(true, |last_table_id| { - last_table_id != last_key.user_key.table_id.table_id + last_table_id != iter_key.user_key.table_id.table_id }) { if let Some(last_table_id) = last_table_id.take() { table_stats_drop.insert(last_table_id, std::mem::take(&mut last_table_stats)); } - last_table_id = Some(last_key.user_key.table_id.table_id); + last_table_id = Some(iter_key.user_key.table_id.table_id); } let target_extended_user_key = PointRange::from_user_key(iter_key.user_key, false); while del_iter.is_valid() && del_iter.key().as_ref().le(&target_extended_user_key) { let event_key = del_iter.key().to_vec(); del_iter.next().await?; - if !task_config.gc_delete_keys { + let new_epoch = del_iter.earliest_epoch(); + if !task_config.gc_delete_keys + && !compaction_filter.should_delete(FullKey::from_user_key( + event_key.left_user_key.as_ref(), + new_epoch, + )) + { sst_builder .add_monotonic_delete(MonotonicDeleteEvent { - new_epoch: del_iter.earliest_epoch(), event_key, + new_epoch, }) .await?; } @@ -809,7 +819,9 @@ where // in our design, frontend avoid to access keys which had be deleted, so we dont // need to consider the epoch when the compaction_filter match (it // means that mv had drop) - if (epoch <= task_config.watermark && task_config.gc_delete_keys && value.is_delete()) + // Because of memtable spill, there may be a PUT key share the same `pure_epoch` with DELETE key. + // Do not assume that "the epoch of keys behind must be smaller than the current key." + if (epoch < task_config.watermark && task_config.gc_delete_keys && value.is_delete()) || (epoch < task_config.watermark && (watermark_can_see_last_key || earliest_range_delete_which_can_see_iter_key <= task_config.watermark)) @@ -829,13 +841,13 @@ where let should_count = match task_config.stats_target_table_ids.as_ref() { Some(target_table_ids) => { - target_table_ids.contains(&last_key.user_key.table_id.table_id) + target_table_ids.contains(&iter_key.user_key.table_id.table_id) } None => true, }; if should_count { last_table_stats.total_key_count -= 1; - last_table_stats.total_key_size -= last_key.encoded_len() as i64; + last_table_stats.total_key_size -= iter_key.encoded_len() as i64; last_table_stats.total_value_size -= iter.value().encoded_len() as i64; } iter.next() @@ -886,6 +898,7 @@ where let end_key_ref = extended_largest_user_key.as_ref(); while del_iter.is_valid() { if !end_key_ref.is_empty() && del_iter.key().ge(&end_key_ref) { + // We do not need to check right bound of delete-range because build would not add it. sst_builder .add_monotonic_delete(MonotonicDeleteEvent { event_key: extended_largest_user_key, @@ -896,12 +909,19 @@ where } let event_key = del_iter.key().to_vec(); del_iter.next().await?; - sst_builder - .add_monotonic_delete(MonotonicDeleteEvent { - new_epoch: del_iter.earliest_epoch(), - event_key, - }) - .await?; + let new_epoch = del_iter.earliest_epoch(); + let drop = compaction_filter.should_delete(FullKey::from_user_key( + event_key.left_user_key.as_ref(), + new_epoch, + )); + if !drop { + sst_builder + .add_monotonic_delete(MonotonicDeleteEvent { + event_key, + new_epoch, + }) + .await?; + } } } if let Some(last_table_id) = last_table_id.take() { @@ -916,59 +936,75 @@ where #[cfg(test)] mod tests { - use std::collections::HashSet; + use std::collections::{HashSet, VecDeque}; use risingwave_common::catalog::TableId; use risingwave_common::util::epoch::test_epoch; - use risingwave_hummock_sdk::key::UserKey; - use risingwave_pb::hummock::InputLevel; + use risingwave_hummock_sdk::key::{TableKey, UserKey}; + use risingwave_hummock_sdk::prost_key_range::KeyRangeExt; + use risingwave_pb::hummock::{InputLevel, PbKeyRange}; use super::*; + use crate::filter_key_extractor::FullKeyFilterKeyExtractor; use crate::hummock::compactor::StateCleanUpCompactionFilter; use crate::hummock::iterator::test_utils::mock_sstable_store; use crate::hummock::test_utils::delete_range::create_monotonic_events; use crate::hummock::test_utils::{default_builder_opt_for_test, gen_test_sstable_impl}; - use crate::hummock::{DeleteRangeTombstone, Xor16FilterBuilder}; + use crate::hummock::{ + DeleteRangeTombstone, SharedComapctorObjectIdManager, Xor16FilterBuilder, + }; + use crate::opts::StorageOpts; #[tokio::test] async fn test_delete_range_aggregator_with_filter() { let sstable_store = mock_sstable_store(); let kv_pairs = vec![]; - let range_tombstones = vec![ - DeleteRangeTombstone::new_for_test( - TableId::new(1), - b"abc".to_vec(), - b"cde".to_vec(), - test_epoch(1), - ), - DeleteRangeTombstone::new_for_test( - TableId::new(2), - b"abc".to_vec(), - b"def".to_vec(), - test_epoch(1), - ), - ]; let mut sstable_info_1 = gen_test_sstable_impl::( default_builder_opt_for_test(), 1, kv_pairs.clone().into_iter(), - range_tombstones.clone(), + vec![ + DeleteRangeTombstone::new_for_test( + TableId::new(1), + b"abc".to_vec(), + b"ccc".to_vec(), + test_epoch(2), + ), + DeleteRangeTombstone::new_for_test( + TableId::new(1), + b"ddd".to_vec(), + b"eee".to_vec(), + test_epoch(2), + ), + ], sstable_store.clone(), CachePolicy::NotFill, ) .await; sstable_info_1.table_ids = vec![1]; - + let tombstone = DeleteRangeTombstone::new_for_test( + TableId::new(2), + b"abc".to_vec(), + b"def".to_vec(), + test_epoch(1), + ); let mut sstable_info_2 = gen_test_sstable_impl::( default_builder_opt_for_test(), 2, - kv_pairs.into_iter(), - range_tombstones.clone(), + vec![( + FullKey::from_user_key( + UserKey::new(TableId::new(1), TableKey(Bytes::copy_from_slice(b"bbb"))), + test_epoch(1), + ), + HummockValue::put(Bytes::copy_from_slice(b"value")), + )] + .into_iter(), + vec![tombstone.clone()], sstable_store.clone(), CachePolicy::NotFill, ) .await; - sstable_info_2.table_ids = vec![2]; + sstable_info_2.table_ids = vec![1, 2]; let compact_task = CompactTask { input_ssts: vec![InputLevel { @@ -977,48 +1013,55 @@ mod tests { table_infos: vec![sstable_info_1, sstable_info_2], }], existing_table_ids: vec![2], + splits: vec![PbKeyRange::inf()], + watermark: 10, ..Default::default() }; - let mut state_clean_up_filter = StateCleanUpCompactionFilter::new(HashSet::from_iter( + let state_clean_up_filter = StateCleanUpCompactionFilter::new(HashSet::from_iter( compact_task.existing_table_ids.clone(), )); - - let sstable_infos = compact_task - .input_ssts - .iter() - .flat_map(|level| level.table_infos.iter()) - .filter(|table_info| { - let table_ids = &table_info.table_ids; - table_ids - .iter() - .any(|table_id| compact_task.existing_table_ids.contains(table_id)) - }) - .cloned() - .collect_vec(); - - let mut iter = ForwardMergeRangeIterator::new(HummockEpoch::MAX); - iter.add_concat_iter(sstable_infos, sstable_store); - - let ret = CompactionDeleteRangeIterator::new(iter) - .get_tombstone_between( - UserKey::::default().as_ref(), - UserKey::::default().as_ref(), + let opts = StorageOpts { + share_buffer_compaction_worker_threads_number: 1, + ..Default::default() + }; + let context = CompactorContext::new_local_compact_context( + Arc::new(opts), + sstable_store.clone(), + Arc::new(CompactorMetrics::unused()), + ); + let runner = CompactorRunner::new( + 0, + context, + compact_task, + Box::new(SharedComapctorObjectIdManager::for_test( + VecDeque::from_iter([5, 6, 7, 8, 9, 10, 11, 12, 13]), + )), + ); + let multi_filter_key_extractor = + Arc::new(FilterKeyExtractorImpl::FullKey(FullKeyFilterKeyExtractor)); + let (_, sst_infos, _) = runner + .run( + state_clean_up_filter, + multi_filter_key_extractor, + Arc::new(TaskProgress::default()), ) .await .unwrap(); - let ret = ret - .into_iter() - .filter(|event| { - !state_clean_up_filter.should_delete(FullKey::from_user_key( - event.event_key.left_user_key.as_ref(), - event.new_epoch, - )) - }) - .collect_vec(); + let sst_infos = sst_infos.into_iter().map(|sst| sst.sst_info).collect_vec(); + let mut ret = vec![]; + for sst_info in sst_infos { + let sst = sstable_store + .sstable(&sst_info, &mut StoreLocalStatistic::default()) + .await + .unwrap(); + ret.append(&mut sst.meta.monotonic_tombstone_events.clone()); + } + let expected_result = create_monotonic_events(vec![tombstone]); assert_eq!( - ret, - create_monotonic_events(vec![range_tombstones[1].clone()]) + ret, expected_result, + "{:?} vs {:?}", + ret[0], expected_result[0], ); } } diff --git a/src/storage/src/hummock/compactor/fast_compactor_runner.rs b/src/storage/src/hummock/compactor/fast_compactor_runner.rs index 828a9750aa41c..92b5e8ee3b835 100644 --- a/src/storage/src/hummock/compactor/fast_compactor_runner.rs +++ b/src/storage/src/hummock/compactor/fast_compactor_runner.rs @@ -34,6 +34,7 @@ use crate::hummock::compactor::task_progress::TaskProgress; use crate::hummock::compactor::{ CompactionStatistics, Compactor, CompactorContext, RemoteBuilderFactory, TaskConfig, }; +use crate::hummock::iterator::SkipWatermarkState; use crate::hummock::multi_builder::{CapacitySplitTableBuilder, TableBuilderFactory}; use crate::hummock::sstable_store::SstableStoreRef; use crate::hummock::value::HummockValue; @@ -92,10 +93,9 @@ impl BlockStreamIterator { None => return Ok(None), Some(ret) => ret, }; - let meta = self.sstable.value().meta.block_metas[self.next_block_index].clone(); + let meta = self.sstable.meta.block_metas[self.next_block_index].clone(); let filter_block = self .sstable - .value() .filter_reader .get_block_raw_filter(self.next_block_index); self.next_block_index += 1; @@ -111,25 +111,25 @@ impl BlockStreamIterator { } fn next_block_smallest(&self) -> &[u8] { - self.sstable.value().meta.block_metas[self.next_block_index] + self.sstable.meta.block_metas[self.next_block_index] .smallest_key .as_ref() } fn next_block_largest(&self) -> &[u8] { - if self.next_block_index + 1 < self.sstable.value().meta.block_metas.len() { - self.sstable.value().meta.block_metas[self.next_block_index + 1] + if self.next_block_index + 1 < self.sstable.meta.block_metas.len() { + self.sstable.meta.block_metas[self.next_block_index + 1] .smallest_key .as_ref() } else { - self.sstable.value().meta.largest_key.as_ref() + self.sstable.meta.largest_key.as_ref() } } fn current_block_largest(&self) -> Vec { - if self.next_block_index < self.sstable.value().meta.block_metas.len() { + if self.next_block_index < self.sstable.meta.block_metas.len() { let mut largest_key = FullKey::decode( - self.sstable.value().meta.block_metas[self.next_block_index] + self.sstable.meta.block_metas[self.next_block_index] .smallest_key .as_ref(), ); @@ -137,7 +137,7 @@ impl BlockStreamIterator { largest_key.epoch_with_gap = EpochWithGap::new_max_epoch(); largest_key.encode() } else { - self.sstable.value().meta.largest_key.clone() + self.sstable.meta.largest_key.clone() } } @@ -145,7 +145,7 @@ impl BlockStreamIterator { match self.iter.as_ref() { Some(iter) => iter.key(), None => FullKey::decode( - self.sstable.value().meta.block_metas[self.next_block_index] + self.sstable.meta.block_metas[self.next_block_index] .smallest_key .as_ref(), ), @@ -153,7 +153,7 @@ impl BlockStreamIterator { } fn is_valid(&self) -> bool { - self.iter.is_some() || self.next_block_index < self.sstable.value().meta.block_metas.len() + self.iter.is_some() || self.next_block_index < self.sstable.meta.block_metas.len() } } @@ -243,7 +243,7 @@ impl ConcatSstableIterator { self.task_progress.inc_num_pending_read_io(); let block_stream = self .sstable_store - .get_stream_for_blocks(sstable.value().id, &sstable.value().meta.block_metas) + .get_stream_for_blocks(sstable.id, &sstable.meta.block_metas) .verbose_instrument_await("stream_iter_get_stream") .await?; @@ -333,9 +333,10 @@ impl CompactorRunner { context.sstable_store, task_progress.clone(), )); + let state = SkipWatermarkState::from_safe_epoch_watermarks(&task.table_watermarks); Self { - executor: CompactTaskExecutor::new(sst_builder, task_config, task_progress), + executor: CompactTaskExecutor::new(sst_builder, task_config, task_progress, state), left, right, task_id: task.task_id, @@ -376,15 +377,7 @@ impl CompactorRunner { } let smallest_key = FullKey::decode(first.current_sstable().next_block_smallest()); - if self - .executor - .last_key - .user_key - .as_ref() - .eq(&smallest_key.user_key) - { - // If the last key is delete tombstone, we can not append the origin block - // because it would cause a deleted key could be see by user again. + if !self.executor.shall_copy_raw_block(&smallest_key) { break; } let smallest_key = smallest_key.to_vec(); @@ -443,7 +436,7 @@ impl CompactorRunner { if rest_data.is_valid() { // compact rest keys of the current block. let sstable_iter = rest_data.sstable_iter.as_mut().unwrap(); - let target_key = FullKey::decode(&sstable_iter.sstable.value().meta.largest_key); + let target_key = FullKey::decode(&sstable_iter.sstable.meta.largest_key); if let Some(iter) = sstable_iter.iter.as_mut() { self.executor.run(iter, target_key).await?; assert!( @@ -464,8 +457,11 @@ impl CompactorRunner { // If the last key is tombstone and it was deleted, the first key of this block must be deleted. So we can not move this block directly. let need_deleted = self.executor.last_key.user_key.eq(&smallest_key.user_key) && self.executor.last_key_is_delete; - if self.executor.builder.need_flush() || need_deleted { - let largest_key = sstable_iter.sstable.value().meta.largest_key.clone(); + if self.executor.builder.need_flush() + || need_deleted + || !self.executor.shall_copy_raw_block(&smallest_key.to_ref()) + { + let largest_key = sstable_iter.sstable.meta.largest_key.clone(); let target_key = FullKey::decode(&largest_key); sstable_iter.init_block_iter(block, block_meta.uncompressed_size as usize)?; let mut iter = sstable_iter.iter.take().unwrap(); @@ -526,12 +522,13 @@ pub struct CompactTaskExecutor { compaction_statistics: CompactionStatistics, last_table_id: Option, last_table_stats: TableStats, - watermark_can_see_last_key: bool, builder: CapacitySplitTableBuilder, task_config: TaskConfig, task_progress: Arc, + state: SkipWatermarkState, last_key_is_delete: bool, progress_key_num: u32, + watermark_can_see_last_key: bool, } impl CompactTaskExecutor { @@ -539,18 +536,20 @@ impl CompactTaskExecutor { builder: CapacitySplitTableBuilder, task_config: TaskConfig, task_progress: Arc, + state: SkipWatermarkState, ) -> Self { Self { builder, task_config, last_key: FullKey::default(), - watermark_can_see_last_key: false, last_key_is_delete: false, compaction_statistics: CompactionStatistics::default(), last_table_id: None, last_table_stats: TableStats::default(), - progress_key_num: 0, task_progress, + state, + progress_key_num: 0, + watermark_can_see_last_key: false, } } @@ -587,6 +586,7 @@ impl CompactTaskExecutor { iter: &mut BlockIterator, target_key: FullKey<&[u8]>, ) -> HummockResult<()> { + self.state.reset_watermark(); while iter.is_valid() && iter.key().le(&target_key) { let is_new_user_key = !self.last_key.is_empty() && iter.key().user_key != self.last_key.user_key.as_ref(); @@ -601,7 +601,9 @@ impl CompactTaskExecutor { self.watermark_can_see_last_key = false; self.last_key_is_delete = false; } - if epoch <= self.task_config.watermark + + // See note in `compactor_runner.rs`. + if epoch < self.task_config.watermark && self.task_config.gc_delete_keys && value.is_delete() { @@ -610,7 +612,10 @@ impl CompactTaskExecutor { } else if epoch < self.task_config.watermark && self.watermark_can_see_last_key { drop = true; } - + if self.state.has_watermark() && self.state.should_delete(&iter.key()) { + drop = true; + self.last_key_is_delete = true; + } if epoch <= self.task_config.watermark { self.watermark_can_see_last_key = true; } @@ -650,4 +655,16 @@ impl CompactTaskExecutor { } Ok(()) } + + pub fn shall_copy_raw_block(&mut self, smallest_key: &FullKey<&[u8]>) -> bool { + if self.last_key_is_delete && self.last_key.user_key.as_ref().eq(&smallest_key.user_key) { + // If the last key is delete tombstone, we can not append the origin block + // because it would cause a deleted key could be see by user again. + return false; + } + if self.state.has_watermark() && self.state.should_delete(smallest_key) { + return false; + } + true + } } diff --git a/src/storage/src/hummock/compactor/iterator.rs b/src/storage/src/hummock/compactor/iterator.rs index fd2a9b1086e6f..9d0a1047dcafe 100644 --- a/src/storage/src/hummock/compactor/iterator.rs +++ b/src/storage/src/hummock/compactor/iterator.rs @@ -359,7 +359,7 @@ impl ConcatSstableIterator { .sstable(table_info, &mut self.stats) .verbose_instrument_await("stream_iter_sstable") .await?; - let block_metas = &sstable.value().meta.block_metas; + let block_metas = &sstable.meta.block_metas; let mut start_index = match seek_key { None => 0, Some(seek_key) => { @@ -398,7 +398,7 @@ impl ConcatSstableIterator { } else { self.task_progress.inc_num_pending_read_io(); let mut sstable_iter = SstableStreamIterator::new( - sstable.value().meta.block_metas.clone(), + sstable.meta.block_metas.clone(), table_info.clone(), self.existing_table_ids.clone(), start_index, @@ -718,7 +718,7 @@ mod tests { .sstable(&iter.sstables[0], &mut iter.stats) .await .unwrap(); - let block_metas = &sst.value().meta.block_metas; + let block_metas = &sst.meta.block_metas; let block_1_smallest_key = block_metas[1].smallest_key.clone(); let block_2_smallest_key = block_metas[2].smallest_key.clone(); // Use block_1_smallest_key as seek key and result in the first KV of block 1. diff --git a/src/storage/src/hummock/compactor/mod.rs b/src/storage/src/hummock/compactor/mod.rs index 1c0c6f78c35ff..b71231ed162a0 100644 --- a/src/storage/src/hummock/compactor/mod.rs +++ b/src/storage/src/hummock/compactor/mod.rs @@ -22,6 +22,7 @@ use risingwave_pb::hummock::report_compaction_task_request::{ }; use risingwave_pb::hummock::{ReportFullScanTaskRequest, ReportVacuumTaskRequest}; use risingwave_rpc_client::GrpcCompactorProxyClient; +use thiserror_ext::AsReport; use tokio::sync::mpsc; use tonic::Request; @@ -49,7 +50,7 @@ use futures::{pin_mut, StreamExt}; pub use iterator::{ConcatSstableIterator, SstableStreamIterator}; use more_asserts::assert_ge; use risingwave_hummock_sdk::table_stats::to_prost_table_stats_map; -use risingwave_hummock_sdk::{HummockCompactionTaskId, LocalSstableInfo}; +use risingwave_hummock_sdk::{compact_task_to_string, HummockCompactionTaskId, LocalSstableInfo}; use risingwave_pb::hummock::compact_task::TaskStatus; use risingwave_pb::hummock::subscribe_compaction_event_request::{ Event as RequestEvent, HeartBeat, PullTask, ReportTask, @@ -66,7 +67,8 @@ use tokio::task::JoinHandle; use tokio::time::Instant; pub use self::compaction_utils::{ - check_compaction_result, CompactionStatistics, RemoteBuilderFactory, TaskConfig, + check_compaction_result, check_flush_result, CompactionStatistics, RemoteBuilderFactory, + TaskConfig, }; pub use self::task_progress::TaskProgress; use super::multi_builder::CapacitySplitTableBuilder; @@ -343,8 +345,8 @@ pub fn start_compactor( Err(e) => { tracing::warn!( - "Subscribing to compaction tasks failed with error: {}. Will retry.", - e + error = %e.as_report(), + "Subscribing to compaction tasks failed with error. Will retry.", ); continue 'start_stream; } @@ -384,7 +386,7 @@ pub fn start_compactor( .expect("Clock may have gone backwards") .as_millis() as u64, }) { - tracing::warn!("Failed to report task progress. {e:?}"); + tracing::warn!(error = %e.as_report(), "Failed to report task progress"); // re subscribe stream continue 'start_stream; } @@ -407,7 +409,7 @@ pub fn start_compactor( .expect("Clock may have gone backwards") .as_millis() as u64, }) { - tracing::warn!("Failed to pull task {e:?}"); + tracing::warn!(error = %e.as_report(), "Failed to pull task"); // re subscribe stream continue 'start_stream; @@ -462,7 +464,7 @@ pub fn start_compactor( let (tx, rx) = tokio::sync::oneshot::channel(); let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); - let (compact_task, table_stats) = match sstable_object_id_manager.add_watermark_object_id(None).await + let ((compact_task, table_stats), _memory_tracker) = match sstable_object_id_manager.add_watermark_object_id(None).await { Ok(tracker_id) => { let sstable_object_id_manager_clone = sstable_object_id_manager.clone(); @@ -472,23 +474,25 @@ pub fn start_compactor( sstable_object_id_manager.remove_watermark_object_id(tracker_id); }, ); - compactor_runner::compact(context, compact_task, rx, Box::new(sstable_object_id_manager.clone()), filter_key_extractor_manager.clone()).await + compactor_runner::compact(context.clone(), compact_task, rx, Box::new(sstable_object_id_manager.clone()), filter_key_extractor_manager.clone()).await }, Err(err) => { - tracing::warn!("Failed to track pending SST object id. {:#?}", err); + tracing::warn!(error = %err.as_report(), "Failed to track pending SST object id"); let mut compact_task = compact_task; compact_task.set_task_status(TaskStatus::TrackSstObjectIdFailed); - (compact_task, HashMap::default()) + ((compact_task, HashMap::default()),None) } }; shutdown.lock().unwrap().remove(&task_id); + let enable_check_compaction_result = context.storage_opts.check_compaction_result; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success; if let Err(e) = request_sender.send(SubscribeCompactionEventRequest { event: Some(RequestEvent::ReportTask( ReportTask { task_id: compact_task.task_id, task_status: compact_task.task_status, - sorted_output_ssts: compact_task.sorted_output_ssts, + sorted_output_ssts: compact_task.sorted_output_ssts.clone(), table_stats_change:to_prost_table_stats_map(table_stats), } )), @@ -497,7 +501,18 @@ pub fn start_compactor( .expect("Clock may have gone backwards") .as_millis() as u64, }) { - tracing::warn!("Failed to report task {task_id:?} . {e:?}"); + tracing::warn!(error = %e.as_report(), "Failed to report task {task_id:?}"); + } + if enable_check_compaction_result && need_check_task { + match check_compaction_result(&compact_task, context.clone()).await { + Err(e) => { + tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}",compact_task.task_id); + }, + Ok(true) => (), + Ok(false) => { + panic!("Failed to pass consistency check for result of compaction task:\n{:?}", compact_task_to_string(&compact_task)); + } + } } } ResponseEvent::VacuumTask(vacuum_task) => { @@ -510,7 +525,7 @@ pub fn start_compactor( Vacuum::report_vacuum_task(vacuum_task, meta_client).await; } Err(e) => { - tracing::warn!("Failed to vacuum task: {:#?}", e) + tracing::warn!(error = %e.as_report(), "Failed to vacuum task") } } } @@ -520,7 +535,7 @@ pub fn start_compactor( Vacuum::report_full_scan_task(object_ids, total_object_count, total_object_size, meta_client).await; } Err(e) => { - tracing::warn!("Failed to iter object: {:#?}", e); + tracing::warn!(error = %e.as_report(), "Failed to iter object"); } } } @@ -605,7 +620,7 @@ pub fn start_shared_compactor( )), }; if let Err(e) = report_heartbeat_client.report_compaction_task(report_compaction_task_request).await{ - tracing::warn!("Failed to report heartbeat {:#?}", e); + tracing::warn!(error = %e.as_report(), "Failed to report heartbeat"); } continue } @@ -654,7 +669,7 @@ pub fn start_shared_compactor( let task_id = compact_task.task_id; shutdown.lock().unwrap().insert(task_id, tx); - let (compact_task, table_stats) = compactor_runner::compact( + let ((compact_task, table_stats), _memory_tracker)= compactor_runner::compact( context.clone(), compact_task, rx, @@ -665,7 +680,7 @@ pub fn start_shared_compactor( shutdown.lock().unwrap().remove(&task_id); let report_compaction_task_request = ReportCompactionTaskRequest { event: Some(ReportCompactionTaskEvent::ReportTask(ReportSharedTask { - compact_task: Some(compact_task), + compact_task: Some(compact_task.clone()), table_stats_change: to_prost_table_stats_map(table_stats), })), }; @@ -674,9 +689,25 @@ pub fn start_shared_compactor( .report_compaction_task(report_compaction_task_request) .await { - Ok(_) => {} - Err(e) => tracing::warn!("Failed to report task {task_id:?} . {e:?}"), + Ok(_) => { + // TODO: remove this method after we have running risingwave cluster with fast compact algorithm stably for a long time. + let enable_check_compaction_result = context.storage_opts.check_compaction_result; + let need_check_task = !compact_task.sorted_output_ssts.is_empty() && compact_task.task_status() == TaskStatus::Success; + if enable_check_compaction_result && need_check_task { + match check_compaction_result(&compact_task, context.clone()).await { + Err(e) => { + tracing::warn!(error = %e.as_report(), "Failed to check compaction task {}", compact_task.task_id); + }, + Ok(true) => (), + Ok(false) => { + panic!("Failed to pass consistency check for result of compaction task:\n{:?}", compact_task_to_string(&compact_task)); + } + } + } + } + Err(e) => tracing::warn!(error = %e.as_report(), "Failed to report task {task_id:?}"), } + } dispatch_compaction_task_request::Task::VacuumTask(vacuum_task) => { match Vacuum::handle_vacuum_task( @@ -691,11 +722,11 @@ pub fn start_shared_compactor( }; match cloned_grpc_proxy_client.report_vacuum_task(report_vacuum_task_request).await { Ok(_) => tracing::info!("Finished vacuuming SSTs"), - Err(e) => tracing::warn!("Failed to report vacuum task: {:#?}", e), + Err(e) => tracing::warn!(error = %e.as_report(), "Failed to report vacuum task"), } } Err(e) => { - tracing::warn!("Failed to vacuum task: {:#?}", e) + tracing::warn!(error = %e.as_report(), "Failed to vacuum task") } } } @@ -714,11 +745,11 @@ pub fn start_shared_compactor( .await { Ok(_) => tracing::info!("Finished full scan SSTs"), - Err(e) => tracing::warn!("Failed to report full scan task: {:#?}", e), + Err(e) => tracing::warn!(error = %e.as_report(), "Failed to report full scan task"), } } Err(e) => { - tracing::warn!("Failed to iter object: {:#?}", e); + tracing::warn!(error = %e.as_report(), "Failed to iter object"); } } } diff --git a/src/storage/src/hummock/compactor/shared_buffer_compact.rs b/src/storage/src/hummock/compactor/shared_buffer_compact.rs index 63ae6ffa301de..4ad2e31136adb 100644 --- a/src/storage/src/hummock/compactor/shared_buffer_compact.rs +++ b/src/storage/src/hummock/compactor/shared_buffer_compact.rs @@ -16,7 +16,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::ops::Bound; use std::sync::Arc; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use futures::future::try_join_all; use futures::{stream, StreamExt, TryFutureExt}; use itertools::Itertools; @@ -24,32 +24,32 @@ use risingwave_common::cache::CachePriority; use risingwave_common::catalog::TableId; use risingwave_common::hash::VirtualNode; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; -use risingwave_hummock_sdk::key::{FullKey, PointRange, TableKey, UserKey}; +use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker, UserKey, EPOCH_LEN}; use risingwave_hummock_sdk::key_range::KeyRange; use risingwave_hummock_sdk::{CompactionGroupId, EpochWithGap, HummockEpoch, LocalSstableInfo}; use risingwave_pb::hummock::compact_task; -use tracing::error; +use thiserror_ext::AsReport; +use tracing::{error, warn}; use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorManager}; use crate::hummock::compactor::compaction_filter::DummyCompactionFilter; use crate::hummock::compactor::context::CompactorContext; -use crate::hummock::compactor::{CompactOutput, Compactor}; +use crate::hummock::compactor::{check_flush_result, CompactOutput, Compactor}; use crate::hummock::event_handler::uploader::UploadTaskPayload; use crate::hummock::event_handler::LocalInstanceId; use crate::hummock::iterator::{ - Forward, ForwardMergeRangeIterator, HummockIterator, OrderedMergeIteratorInner, + Forward, ForwardMergeRangeIterator, HummockIterator, MergeIterator, UserIterator, }; use crate::hummock::shared_buffer::shared_buffer_batch::{ - SharedBufferBatch, SharedBufferBatchInner, SharedBufferVersionedEntry, + SharedBufferBatch, SharedBufferBatchInner, SharedBufferKeyEntry, VersionedSharedBufferValue, }; use crate::hummock::utils::MemoryTracker; -use crate::hummock::value::HummockValue; use crate::hummock::{ BlockedXor16FilterBuilder, CachePolicy, CompactionDeleteRangeIterator, GetObjectId, - HummockError, HummockResult, MonotonicDeleteEvent, SstableBuilderOptions, - SstableObjectIdManagerRef, + HummockError, HummockResult, SstableBuilderOptions, SstableObjectIdManagerRef, }; use crate::mem_table::ImmutableMemtable; +use crate::opts::StorageOpts; const GC_DELETE_KEYS_FOR_FLUSH: bool = false; const GC_WATERMARK_FOR_FLUSH: u64 = 0; @@ -125,7 +125,6 @@ async fn compact_shared_buffer( .map(|imm| imm.table_id.table_id) .dedup() .collect(); - assert!(!existing_table_ids.is_empty()); let multi_filter_key_extractor = filter_key_extractor_manager @@ -136,8 +135,6 @@ async fn compact_shared_buffer( } let multi_filter_key_extractor = Arc::new(multi_filter_key_extractor); - let mut size_and_start_user_keys = vec![]; - let mut compact_data_size = 0; payload.retain(|imm| { let ret = existing_table_ids.contains(&imm.table_id.table_id); if !ret { @@ -148,81 +145,10 @@ async fn compact_shared_buffer( } ret }); - let mut total_key_count = 0; - for imm in &payload { - total_key_count += imm.kv_count(); - let data_size = { - // calculate encoded bytes of key var length - (imm.kv_count() * 8 + imm.size()) as u64 - }; - compact_data_size += data_size; - size_and_start_user_keys.push((data_size, imm.start_user_key())); - } - size_and_start_user_keys.sort(); - let mut splits = Vec::with_capacity(size_and_start_user_keys.len()); - splits.push(KeyRange::new(Bytes::new(), Bytes::new())); - let mut key_split_append = |key_before_last: &Bytes| { - splits.last_mut().unwrap().right = key_before_last.clone(); - splits.push(KeyRange::new(key_before_last.clone(), Bytes::new())); - }; - let sstable_size = (context.storage_opts.sstable_size_mb as u64) << 20; - let parallel_compact_size = (context.storage_opts.parallel_compact_size_mb as u64) << 20; - let parallelism = std::cmp::min( - context.storage_opts.share_buffers_sync_parallelism as u64, - size_and_start_user_keys.len() as u64, - ); - let sub_compaction_data_size = if compact_data_size > parallel_compact_size && parallelism > 1 { - compact_data_size / parallelism - } else { - compact_data_size - }; - // mul 1.2 for other extra memory usage. - let mut sub_compaction_sstable_size = - std::cmp::min(sstable_size, sub_compaction_data_size * 6 / 5); - let mut split_weight_by_vnode = 0; - if existing_table_ids.len() > 1 { - if parallelism > 1 && compact_data_size > sstable_size { - let mut last_buffer_size = 0; - let mut last_user_key = UserKey::default(); - for (data_size, user_key) in size_and_start_user_keys { - if last_buffer_size >= sub_compaction_data_size - && last_user_key.as_ref() != user_key - { - last_user_key.set(user_key); - key_split_append( - &FullKey { - user_key, - epoch_with_gap: EpochWithGap::new_max_epoch(), - } - .encode() - .into(), - ); - last_buffer_size = data_size; - } else { - last_user_key.set(user_key); - last_buffer_size += data_size; - } - } - } - } else { - let mut vnodes = vec![]; - for imm in &payload { - vnodes.extend(imm.collect_vnodes()); - } - vnodes.sort(); - vnodes.dedup(); - const MIN_SSTABLE_SIZE: u64 = 16 * 1024 * 1024; - if compact_data_size >= MIN_SSTABLE_SIZE && !vnodes.is_empty() { - let mut avg_vnode_size = compact_data_size / (vnodes.len() as u64); - split_weight_by_vnode = VirtualNode::COUNT; - while avg_vnode_size < MIN_SSTABLE_SIZE && split_weight_by_vnode > 0 { - split_weight_by_vnode /= 2; - avg_vnode_size *= 2; - } - sub_compaction_sstable_size = compact_data_size; - } - } + let total_key_count = payload.iter().map(|imm| imm.key_count()).sum::(); + let (splits, sub_compaction_sstable_size, split_weight_by_vnode) = + generate_splits(&payload, &existing_table_ids, context.storage_opts.as_ref()); let parallelism = splits.len(); let mut compact_success = true; let mut output_ssts = Vec::with_capacity(parallelism); @@ -231,7 +157,7 @@ async fn compact_shared_buffer( let table_vnode_partition = if existing_table_ids.len() == 1 { let table_id = existing_table_ids.iter().next().unwrap(); - vec![(*table_id, split_weight_by_vnode as u32)] + vec![(*table_id, split_weight_by_vnode)] .into_iter() .collect() } else { @@ -248,19 +174,16 @@ async fn compact_shared_buffer( Box::new(sstable_object_id_manager.clone()), ); let mut forward_iters = Vec::with_capacity(payload.len()); - let mut del_iter = ForwardMergeRangeIterator::new(HummockEpoch::MAX); for imm in &payload { forward_iters.push(imm.clone().into_forward_iter()); - del_iter.add_batch_iter(imm.delete_range_iter()); } let compaction_executor = context.compaction_executor.clone(); let multi_filter_key_extractor = multi_filter_key_extractor.clone(); let handle = compaction_executor.spawn(async move { compactor .run( - OrderedMergeIteratorInner::new(forward_iters), + MergeIterator::new(forward_iters), multi_filter_key_extractor, - CompactionDeleteRangeIterator::new(del_iter), ) .await }); @@ -276,14 +199,14 @@ async fn compact_shared_buffer( } Ok(Err(e)) => { compact_success = false; - tracing::warn!("Shared Buffer Compaction failed with error: {:#?}", e); + tracing::warn!(error = %e.as_report(), "Shared Buffer Compaction failed with error"); err = Some(e); } Err(e) => { compact_success = false; tracing::warn!( - "Shared Buffer Compaction failed with future error: {:#?}", - e + error = %e.as_report(), + "Shared Buffer Compaction failed with future error", ); err = Some(HummockError::compaction_executor( "failed while execute in tokio", @@ -297,15 +220,58 @@ async fn compact_shared_buffer( if compact_success { let mut level0 = Vec::with_capacity(parallelism); + let mut sst_infos = vec![]; for (_, ssts, _) in output_ssts { for sst_info in &ssts { context .compactor_metrics .write_build_l0_bytes .inc_by(sst_info.file_size()); + sst_infos.push(sst_info.sst_info.clone()); } level0.extend(ssts); } + if context.storage_opts.check_compaction_result { + let compaction_executor = context.compaction_executor.clone(); + let mut forward_iters = Vec::with_capacity(payload.len()); + let del_iter = ForwardMergeRangeIterator::new(HummockEpoch::MAX); + for imm in &payload { + if !existing_table_ids.contains(&imm.table_id.table_id) { + continue; + } + forward_iters.push(imm.clone().into_forward_iter()); + } + let iter = MergeIterator::new(forward_iters); + let left_iter = UserIterator::new( + iter, + (Bound::Unbounded, Bound::Unbounded), + u64::MAX, + 0, + None, + del_iter, + ); + compaction_executor.spawn(async move { + match check_flush_result( + left_iter, + Vec::from_iter(existing_table_ids.iter().cloned()), + sst_infos, + context, + ) + .await + { + Err(e) => { + tracing::warn!(error = %e.as_report(), "Failed check flush result of memtable"); + } + Ok(true) => (), + Ok(false) => { + panic!( + "failed to check flush result consistency of state-table {:?}", + existing_table_ids + ); + } + } + }); + } Ok(level0) } else { Err(err.unwrap()) @@ -318,162 +284,195 @@ pub async fn merge_imms_in_memory( instance_id: LocalInstanceId, imms: Vec, memory_tracker: Option, -) -> HummockResult { - let mut kv_count = 0; +) -> ImmutableMemtable { let mut epochs = vec![]; let mut merged_size = 0; - let mut merged_imm_ids = Vec::with_capacity(imms.len()); - - let mut smallest_table_key = BytesMut::new(); - let mut smallest_empty = true; - let mut largest_table_key = Bound::Included(Bytes::new()); + assert!(imms.iter().rev().map(|imm| imm.batch_id()).is_sorted()); + let max_imm_id = imms[0].batch_id(); let mut imm_iters = Vec::with_capacity(imms.len()); - let mut del_iter = ForwardMergeRangeIterator::new(HummockEpoch::MAX); + let key_count = imms.iter().map(|imm| imm.key_count()).sum(); + let value_count = imms.iter().map(|imm| imm.value_count()).sum(); for imm in imms { - assert!( - imm.kv_count() > 0 || imm.has_range_tombstone(), - "imm should not be empty" - ); + assert!(imm.key_count() > 0, "imm should not be empty"); assert_eq!( table_id, imm.table_id(), "should only merge data belonging to the same table" ); - merged_imm_ids.push(imm.batch_id()); epochs.push(imm.min_epoch()); - kv_count += imm.kv_count(); merged_size += imm.size(); - del_iter.add_batch_iter(imm.delete_range_iter()); - - if smallest_empty || smallest_table_key.as_ref().gt(imm.raw_smallest_key()) { - smallest_table_key.clear(); - smallest_table_key.extend_from_slice(imm.raw_smallest_key()); - smallest_empty = false; - } - let imm_raw_largest_key = imm.raw_largest_key(); - if match (&largest_table_key, imm_raw_largest_key) { - (_, Bound::Unbounded) => true, - (Bound::Included(x), Bound::Included(y)) | (Bound::Included(x), Bound::Excluded(y)) => { - x < y - } - (Bound::Excluded(x), Bound::Included(y)) | (Bound::Excluded(x), Bound::Excluded(y)) => { - x <= y - } - (Bound::Unbounded, _) => false, - } { - largest_table_key = imm_raw_largest_key.as_ref().cloned(); - } imm_iters.push(imm.into_forward_iter()); } - let mut del_iter = CompactionDeleteRangeIterator::new(del_iter); - del_iter.rewind().await?; epochs.sort(); // use merge iterator to merge input imms - let mut mi = OrderedMergeIteratorInner::new(imm_iters); - mi.rewind().await?; - let mut items = Vec::with_capacity(kv_count); - while mi.is_valid() { - let (key, (epoch, value)) = mi.current_item(); - items.push(((key, value), epoch)); - mi.next().await?; - } + let mut mi = MergeIterator::new(imm_iters); + mi.rewind_no_await(); + assert!(mi.is_valid()); - let mut merged_payload: Vec = Vec::new(); - let mut pivot = items - .first() - .map(|((k, _), _)| k.clone()) - .unwrap_or_default(); - let mut monotonic_tombstone_events = vec![]; - let target_extended_user_key = - PointRange::from_user_key(UserKey::new(table_id, TableKey(pivot.as_ref())), false); - while del_iter.is_valid() && del_iter.key().le(&target_extended_user_key) { - let event_key = del_iter.key().to_vec(); - del_iter.next().await?; - monotonic_tombstone_events.push(MonotonicDeleteEvent { - event_key, - new_epoch: del_iter.earliest_epoch(), - }); - } + let first_item_key = mi.current_key_entry().key.clone(); + + let mut merged_entries: Vec = Vec::with_capacity(key_count); + let mut values: Vec = Vec::with_capacity(value_count); - let mut versions: Vec<(EpochWithGap, HummockValue)> = Vec::new(); - - let mut pivot_last_delete_epoch = HummockEpoch::MAX; - - for ((key, value), epoch) in items { - assert!(key >= pivot, "key should be in ascending order"); - if key != pivot { - merged_payload.push((pivot, versions)); - pivot = key; - pivot_last_delete_epoch = HummockEpoch::MAX; - versions = vec![]; - let target_extended_user_key = - PointRange::from_user_key(UserKey::new(table_id, TableKey(pivot.as_ref())), false); - while del_iter.is_valid() && del_iter.key().le(&target_extended_user_key) { - let event_key = del_iter.key().to_vec(); - del_iter.next().await?; - monotonic_tombstone_events.push(MonotonicDeleteEvent { - event_key, - new_epoch: del_iter.earliest_epoch(), + merged_entries.push(SharedBufferKeyEntry { + key: first_item_key.clone(), + value_offset: 0, + }); + + // Use first key, max epoch to initialize the tracker to ensure that the check first call to full_key_tracker.observe will succeed + let mut full_key_tracker = FullKeyTracker::::new(FullKey::new_with_gap_epoch( + table_id, + first_item_key, + EpochWithGap::new_max_epoch(), + )); + + while mi.is_valid() { + let key_entry = mi.current_key_entry(); + let user_key = UserKey { + table_id, + table_key: key_entry.key.clone(), + }; + if full_key_tracker + .observe_multi_version( + user_key, + key_entry + .new_values + .iter() + .map(|(epoch_with_gap, _)| *epoch_with_gap), + ) + .is_some() + { + let last_entry = merged_entries.last_mut().expect("non-empty"); + if last_entry.value_offset == values.len() { + warn!(key = ?last_entry.key, "key has no value in imm compact. skipped"); + last_entry.key = full_key_tracker.latest_user_key().table_key.clone(); + } else { + // Record kv entries + merged_entries.push(SharedBufferKeyEntry { + key: full_key_tracker.latest_user_key().table_key.clone(), + value_offset: values.len(), }); } } - let earliest_range_delete_which_can_see_key = - del_iter.earliest_delete_since(epoch.pure_epoch()); - if value.is_delete() { - pivot_last_delete_epoch = epoch.pure_epoch(); - } else if earliest_range_delete_which_can_see_key < pivot_last_delete_epoch { - debug_assert!( - epoch.pure_epoch() < earliest_range_delete_which_can_see_key - && earliest_range_delete_which_can_see_key < pivot_last_delete_epoch - ); - pivot_last_delete_epoch = earliest_range_delete_which_can_see_key; - // In each merged immutable memtable, since a union set of delete ranges is constructed - // and thus original delete ranges are replaced with the union set and not - // used in read, we lose exact information about whether a key is deleted by - // a delete range in the merged imm which it belongs to. Therefore we need - // to construct a corresponding delete key to represent this. - versions.push(( - EpochWithGap::new_from_epoch(earliest_range_delete_which_can_see_key), - HummockValue::Delete, - )); - } - versions.push((epoch, value)); - } - while del_iter.is_valid() { - let event_key = del_iter.key().to_vec(); - del_iter.next().await?; - monotonic_tombstone_events.push(MonotonicDeleteEvent { - event_key, - new_epoch: del_iter.earliest_epoch(), - }); - } - - // process the last key - if !versions.is_empty() { - merged_payload.push((pivot, versions)); + values.extend( + key_entry + .new_values + .iter() + .map(|(epoch_with_gap, value)| (*epoch_with_gap, value.clone())), + ); + mi.advance_peek_to_next_key(); + // Since there is no blocking point in this method, but it is cpu intensive, we call this method + // to do cooperative scheduling + tokio::task::consume_budget().await; } - drop(del_iter); - - Ok(SharedBufferBatch { + SharedBufferBatch { inner: Arc::new(SharedBufferBatchInner::new_with_multi_epoch_batches( epochs, - merged_payload, - smallest_table_key.freeze(), - largest_table_key, - kv_count, - merged_imm_ids, - monotonic_tombstone_events, + merged_entries, + values, merged_size, + max_imm_id, memory_tracker, )), table_id, instance_id, - }) + } +} + +/// Based on the incoming payload and opts, calculate the sharding method and sstable size of shared buffer compaction. +fn generate_splits( + payload: &UploadTaskPayload, + existing_table_ids: &HashSet, + storage_opts: &StorageOpts, +) -> (Vec, u64, u32) { + let mut size_and_start_user_keys = vec![]; + let mut compact_data_size = 0; + for imm in payload { + let data_size = { + // calculate encoded bytes of key var length + (imm.value_count() * EPOCH_LEN + imm.size()) as u64 + }; + compact_data_size += data_size; + size_and_start_user_keys.push((data_size, imm.start_user_key())); + } + size_and_start_user_keys.sort_by(|a, b| a.1.cmp(&b.1)); + let mut splits = Vec::with_capacity(size_and_start_user_keys.len()); + splits.push(KeyRange::new(Bytes::new(), Bytes::new())); + let mut key_split_append = |key_before_last: &Bytes| { + splits.last_mut().unwrap().right = key_before_last.clone(); + splits.push(KeyRange::new(key_before_last.clone(), Bytes::new())); + }; + let sstable_size = (storage_opts.sstable_size_mb as u64) << 20; + let parallel_compact_size = (storage_opts.parallel_compact_size_mb as u64) << 20; + let parallelism = std::cmp::min( + storage_opts.share_buffers_sync_parallelism as u64, + size_and_start_user_keys.len() as u64, + ); + let sub_compaction_data_size = if compact_data_size > parallel_compact_size && parallelism > 1 { + compact_data_size / parallelism + } else { + compact_data_size + }; + + let mut vnode_partition_count = 0; + if existing_table_ids.len() > 1 { + if parallelism > 1 && compact_data_size > sstable_size { + let mut last_buffer_size = 0; + let mut last_user_key = UserKey::default(); + for (data_size, user_key) in size_and_start_user_keys { + if last_buffer_size >= sub_compaction_data_size + && last_user_key.as_ref() != user_key + { + last_user_key.set(user_key); + key_split_append( + &FullKey { + user_key, + epoch_with_gap: EpochWithGap::new_max_epoch(), + } + .encode() + .into(), + ); + last_buffer_size = data_size; + } else { + last_user_key.set(user_key); + last_buffer_size += data_size; + } + } + } + } else { + // Collect vnodes in imm + let mut vnodes = vec![]; + for imm in payload { + vnodes.extend(imm.collect_vnodes()); + } + vnodes.sort(); + vnodes.dedup(); + + // Based on the estimated `vnode_avg_size`, calculate the required `vnode_partition_count` to avoid small files and further align + const MIN_SSTABLE_SIZE: u64 = 16 * 1024 * 1024; + if compact_data_size >= MIN_SSTABLE_SIZE && !vnodes.is_empty() { + let mut avg_vnode_size = compact_data_size / (vnodes.len() as u64); + vnode_partition_count = VirtualNode::COUNT; + while avg_vnode_size < MIN_SSTABLE_SIZE && vnode_partition_count > 0 { + vnode_partition_count /= 2; + avg_vnode_size *= 2; + } + } + } + + // mul 1.2 for other extra memory usage. + // Ensure that the size of each sstable is still less than `sstable_size` after optimization to avoid generating a huge size sstable which will affect the object store + let sub_compaction_sstable_size = std::cmp::min(sstable_size, sub_compaction_data_size * 6 / 5); + ( + splits, + sub_compaction_sstable_size, + vnode_partition_count as u32, + ) } pub struct SharedBufferCompactRunner { @@ -519,7 +518,6 @@ impl SharedBufferCompactRunner { self, iter: impl HummockIterator, filter_key_extractor: Arc, - del_iter: CompactionDeleteRangeIterator, ) -> HummockResult { let dummy_compaction_filter = DummyCompactionFilter {}; let (ssts, table_stats_map) = self @@ -527,7 +525,9 @@ impl SharedBufferCompactRunner { .compact_key_range( iter, dummy_compaction_filter, - del_iter, + CompactionDeleteRangeIterator::new(ForwardMergeRangeIterator::new( + HummockEpoch::MAX, + )), filter_key_extractor, None, None, @@ -537,3 +537,102 @@ impl SharedBufferCompactRunner { Ok((self.split_index, ssts, table_stats_map)) } } + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + + use bytes::Bytes; + use risingwave_common::catalog::TableId; + use risingwave_common::hash::VirtualNode; + use risingwave_common::util::epoch::test_epoch; + use risingwave_hummock_sdk::key::{prefix_slice_with_vnode, TableKey}; + + use crate::hummock::compactor::shared_buffer_compact::generate_splits; + use crate::hummock::value::HummockValue; + use crate::mem_table::ImmutableMemtable; + use crate::opts::StorageOpts; + + fn generate_key(key: &str) -> TableKey { + TableKey(prefix_slice_with_vnode( + VirtualNode::from_index(1), + key.as_bytes(), + )) + } + + #[tokio::test] + async fn test_generate_splits_in_order() { + let imm1 = ImmutableMemtable::build_shared_buffer_batch_for_test( + test_epoch(3), + 0, + vec![( + generate_key("dddd"), + HummockValue::put(Bytes::from_static(b"v3")), + )], + 1024 * 1024, + TableId::new(1), + ); + let imm2 = ImmutableMemtable::build_shared_buffer_batch_for_test( + test_epoch(3), + 0, + vec![( + generate_key("abb"), + HummockValue::put(Bytes::from_static(b"v3")), + )], + (1024 + 256) * 1024, + TableId::new(1), + ); + + let imm3 = ImmutableMemtable::build_shared_buffer_batch_for_test( + test_epoch(2), + 0, + vec![( + generate_key("abc"), + HummockValue::put(Bytes::from_static(b"v2")), + )], + (1024 + 512) * 1024, + TableId::new(1), + ); + let imm4 = ImmutableMemtable::build_shared_buffer_batch_for_test( + test_epoch(3), + 0, + vec![( + generate_key("aaa"), + HummockValue::put(Bytes::from_static(b"v3")), + )], + (1024 + 512) * 1024, + TableId::new(1), + ); + + let imm5 = ImmutableMemtable::build_shared_buffer_batch_for_test( + test_epoch(3), + 0, + vec![( + generate_key("aaa"), + HummockValue::put(Bytes::from_static(b"v3")), + )], + (1024 + 256) * 1024, + TableId::new(2), + ); + + let storage_opts = StorageOpts { + share_buffers_sync_parallelism: 3, + parallel_compact_size_mb: 1, + sstable_size_mb: 1, + ..Default::default() + }; + let payload = vec![imm1, imm2, imm3, imm4, imm5]; + let (splits, _sstable_capacity, vnode) = + generate_splits(&payload, &HashSet::from_iter([1, 2]), &storage_opts); + assert_eq!( + splits.len(), + storage_opts.share_buffers_sync_parallelism as usize + ); + assert_eq!(vnode, 0); + for i in 1..splits.len() { + assert_eq!(splits[i].left, splits[i - 1].right); + assert!(splits[i].left > splits[i - 1].left); + assert!(splits[i].right.is_empty() || splits[i].left < splits[i].right); + } + } +} diff --git a/src/storage/src/hummock/error.rs b/src/storage/src/hummock/error.rs index e3dcf712f0e51..d1b662ce148bd 100644 --- a/src/storage/src/hummock/error.rs +++ b/src/storage/src/hummock/error.rs @@ -14,6 +14,7 @@ use risingwave_object_store::object::ObjectError; use thiserror::Error; +use thiserror_ext::AsReport; use tokio::sync::oneshot::error::RecvError; // TODO(error-handling): should prefer use error types than strings. @@ -152,7 +153,7 @@ impl HummockError { impl From for HummockError { fn from(error: prost::DecodeError) -> Self { - HummockErrorInner::DecodeError(error.to_string()).into() + HummockErrorInner::DecodeError(error.to_report_string()).into() } } diff --git a/src/storage/src/hummock/event_handler/hummock_event_handler.rs b/src/storage/src/hummock/event_handler/hummock_event_handler.rs index 1fefdb0f5ea58..9daac68f018a5 100644 --- a/src/storage/src/hummock/event_handler/hummock_event_handler.rs +++ b/src/storage/src/hummock/event_handler/hummock_event_handler.rs @@ -19,11 +19,15 @@ use std::sync::Arc; use arc_swap::ArcSwap; use await_tree::InstrumentAwait; +use itertools::Itertools; use parking_lot::RwLock; use prometheus::core::{AtomicU64, GenericGauge}; +use risingwave_hummock_sdk::compaction_group::hummock_version_ext::SstDeltaInfo; use risingwave_hummock_sdk::{HummockEpoch, LocalSstableInfo}; +use thiserror_ext::AsReport; use tokio::spawn; -use tokio::sync::{mpsc, oneshot}; +use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; +use tokio::sync::oneshot; use tracing::{debug, error, info, trace, warn}; use super::refiller::{CacheRefillConfig, CacheRefiller}; @@ -31,18 +35,22 @@ use super::{LocalInstanceGuard, LocalInstanceId, ReadVersionMappingType}; use crate::filter_key_extractor::FilterKeyExtractorManager; use crate::hummock::compactor::{compact, CompactorContext}; use crate::hummock::conflict_detector::ConflictDetector; -use crate::hummock::event_handler::refiller::CacheRefillerEvent; +use crate::hummock::event_handler::refiller::{CacheRefillerEvent, SpawnRefillTask}; use crate::hummock::event_handler::uploader::{ - HummockUploader, SyncedData, UploadTaskInfo, UploadTaskPayload, UploaderEvent, + default_spawn_merging_task, HummockUploader, SpawnMergingTask, SpawnUploadTask, SyncedData, + UploadTaskInfo, UploadTaskPayload, UploaderEvent, +}; +use crate::hummock::event_handler::{ + HummockEvent, HummockReadVersionRef, HummockVersionUpdate, ReadOnlyReadVersionMapping, + ReadOnlyRwLockRef, }; -use crate::hummock::event_handler::{HummockEvent, HummockVersionUpdate}; use crate::hummock::local_version::pinned_version::PinnedVersion; use crate::hummock::store::version::{ HummockReadVersion, StagingData, StagingSstableInfo, VersionUpdate, }; use crate::hummock::utils::validate_table_key_range; use crate::hummock::{ - HummockError, HummockResult, MemoryLimiter, SstableObjectIdManager, TrackerId, + HummockError, HummockResult, MemoryLimiter, SstableObjectIdManager, SstableStoreRef, TrackerId, }; use crate::monitor::HummockStateStoreMetrics; use crate::opts::StorageOpts; @@ -111,10 +119,13 @@ impl BufferTracker { } pub struct HummockEventHandler { - hummock_event_tx: mpsc::UnboundedSender, - hummock_event_rx: mpsc::UnboundedReceiver, + hummock_event_tx: UnboundedSender, + hummock_event_rx: UnboundedReceiver, + version_update_rx: UnboundedReceiver, pending_sync_requests: BTreeMap>>, - read_version_mapping: Arc, + read_version_mapping: Arc>, + /// A copy of `read_version_mapping` but owned by event handler + local_read_version_mapping: HashMap, version_update_notifier_tx: Arc>, pinned_version: Arc>, @@ -125,7 +136,7 @@ pub struct HummockEventHandler { last_instance_id: LocalInstanceId, - sstable_object_id_manager: Arc, + sstable_object_id_manager: Option>, } async fn flush_imms( @@ -140,7 +151,7 @@ async fn flush_imms( .add_watermark_object_id(Some(*epoch)) .await .inspect_err(|e| { - error!("unable to set watermark sst id. epoch: {}, {:?}", epoch, e); + error!(epoch, error = %e.as_report(), "unable to set watermark sst id"); }); } compact( @@ -156,31 +167,22 @@ async fn flush_imms( impl HummockEventHandler { pub fn new( - hummock_event_tx: mpsc::UnboundedSender, - hummock_event_rx: mpsc::UnboundedReceiver, + version_update_rx: UnboundedReceiver, pinned_version: PinnedVersion, compactor_context: CompactorContext, filter_key_extractor_manager: FilterKeyExtractorManager, sstable_object_id_manager: Arc, state_store_metrics: Arc, - cache_refill_config: CacheRefillConfig, ) -> Self { - let (version_update_notifier_tx, _) = - tokio::sync::watch::channel(pinned_version.max_committed_epoch()); - let version_update_notifier_tx = Arc::new(version_update_notifier_tx); - let read_version_mapping = Arc::new(RwLock::new(HashMap::default())); - let buffer_tracker = BufferTracker::from_storage_opts( - &compactor_context.storage_opts, - state_store_metrics.uploader_uploading_task_size.clone(), - ); - let write_conflict_detector = - ConflictDetector::new_from_config(&compactor_context.storage_opts); - let sstable_store = compactor_context.sstable_store.clone(); let upload_compactor_context = compactor_context.clone(); let cloned_sstable_object_id_manager = sstable_object_id_manager.clone(); - let uploader = HummockUploader::new( + Self::new_inner( + version_update_rx, + pinned_version, + Some(sstable_object_id_manager), + compactor_context.sstable_store.clone(), state_store_metrics, - pinned_version.clone(), + &compactor_context.storage_opts, Arc::new(move |payload, task_info| { spawn(flush_imms( payload, @@ -190,20 +192,57 @@ impl HummockEventHandler { cloned_sstable_object_id_manager.clone(), )) }), + default_spawn_merging_task(compactor_context.compaction_executor.clone()), + CacheRefiller::default_spawn_refill_task(), + ) + } + + fn new_inner( + version_update_rx: UnboundedReceiver, + pinned_version: PinnedVersion, + sstable_object_id_manager: Option>, + sstable_store: SstableStoreRef, + state_store_metrics: Arc, + storage_opts: &StorageOpts, + spawn_upload_task: SpawnUploadTask, + spawn_merging_task: SpawnMergingTask, + spawn_refill_task: SpawnRefillTask, + ) -> Self { + let (hummock_event_tx, hummock_event_rx) = unbounded_channel(); + let (version_update_notifier_tx, _) = + tokio::sync::watch::channel(pinned_version.max_committed_epoch()); + let version_update_notifier_tx = Arc::new(version_update_notifier_tx); + let read_version_mapping = Arc::new(RwLock::new(HashMap::default())); + let buffer_tracker = BufferTracker::from_storage_opts( + storage_opts, + state_store_metrics.uploader_uploading_task_size.clone(), + ); + let write_conflict_detector = ConflictDetector::new_from_config(storage_opts); + + let uploader = HummockUploader::new( + state_store_metrics, + pinned_version.clone(), + spawn_upload_task, + spawn_merging_task, buffer_tracker, - &compactor_context.storage_opts, - compactor_context.compaction_executor.clone(), + storage_opts, + ); + let refiller = CacheRefiller::new( + CacheRefillConfig::from_storage_opts(storage_opts), + sstable_store, + spawn_refill_task, ); - let refiller = CacheRefiller::new(cache_refill_config, sstable_store); Self { hummock_event_tx, hummock_event_rx, + version_update_rx, pending_sync_requests: Default::default(), version_update_notifier_tx, pinned_version: Arc::new(ArcSwap::from_pointee(pinned_version)), write_conflict_detector, read_version_mapping, + local_read_version_mapping: Default::default(), uploader, refiller, last_instance_id: 0, @@ -219,8 +258,12 @@ impl HummockEventHandler { self.pinned_version.clone() } - pub fn read_version_mapping(&self) -> Arc { - self.read_version_mapping.clone() + pub fn read_version_mapping(&self) -> ReadOnlyReadVersionMapping { + ReadOnlyRwLockRef::new(self.read_version_mapping.clone()) + } + + pub fn event_sender(&self) -> UnboundedSender { + self.hummock_event_tx.clone() } pub fn buffer_tracker(&self) -> &BufferTracker { @@ -243,7 +286,7 @@ impl HummockEventHandler { // older data first .rev() .for_each(|staging_sstable_info| { - Self::for_each_read_version(&self.read_version_mapping, |read_version| { + self.for_each_read_version(|read_version| { read_version.update(VersionUpdate::Staging(StagingData::Sst( staging_sstable_info.clone(), ))) @@ -281,21 +324,15 @@ impl HummockEventHandler { /// This function will be performed under the protection of the `read_version_mapping` read /// lock, and add write lock on each `read_version` operation - fn for_each_read_version(read_version: &Arc, mut f: F) - where - F: FnMut(&mut HummockReadVersion), - { - let read_version_mapping_guard = read_version.read(); - - read_version_mapping_guard + fn for_each_read_version(&self, mut f: impl FnMut(&mut HummockReadVersion)) { + self.local_read_version_mapping .values() - .flat_map(HashMap::values) - .for_each(|read_version| f(read_version.write().deref_mut())); + .for_each(|read_version: &HummockReadVersionRef| f(read_version.write().deref_mut())); } fn handle_data_spilled(&mut self, staging_sstable_info: StagingSstableInfo) { // todo: do some prune for version update - Self::for_each_read_version(&self.read_version_mapping, |read_version| { + self.for_each_read_version(|read_version| { trace!("data_spilled. SST size {}", staging_sstable_info.imm_size()); read_version.update(VersionUpdate::Staging(StagingData::Sst( staging_sstable_info.clone(), @@ -367,15 +404,74 @@ impl HummockEventHandler { } } - fn handle_clear(&mut self, notifier: oneshot::Sender<()>) { + async fn handle_clear(&mut self, notifier: oneshot::Sender<()>, prev_epoch: u64) { info!( - "handle clear event. max_committed_epoch: {}, max_synced_epoch: {}, max_sealed_epoch: {}", - self.uploader.max_committed_epoch(), - self.uploader.max_synced_epoch(), - self.uploader.max_sealed_epoch(), + prev_epoch, + max_committed_epoch = self.uploader.max_committed_epoch(), + max_synced_epoch = self.uploader.max_synced_epoch(), + max_sealed_epoch = self.uploader.max_sealed_epoch(), + "handle clear event" ); + self.uploader.clear(); + let current_version = self.uploader.hummock_version(); + + if current_version.max_committed_epoch() < prev_epoch { + let mut latest_version = if let Some(CacheRefillerEvent { + pinned_version, + new_pinned_version, + }) = self.refiller.clear() + { + assert_eq!( + current_version.id(), + pinned_version.id(), + "refiller earliest version {:?} not equal to current version {:?}", + pinned_version.version(), + current_version.version() + ); + + info!( + prev_epoch, + current_mce = current_version.max_committed_epoch(), + refiller_mce = new_pinned_version.max_committed_epoch(), + "refiller is clear in recovery" + ); + + Some(new_pinned_version) + } else { + None + }; + + while let latest_version_ref = latest_version.as_ref().unwrap_or(current_version) + && latest_version_ref.max_committed_epoch() < prev_epoch + { + let version_update = self + .version_update_rx + .recv() + .await + .expect("should not be empty"); + latest_version = Some(Self::resolve_version_update_info( + latest_version_ref.clone(), + version_update, + None, + )); + } + + self.apply_version_update( + current_version.clone(), + latest_version.expect("must have some version update to raise the mce"), + ); + } + + assert!(self.uploader.max_committed_epoch() >= prev_epoch); + if self.uploader.max_committed_epoch() > prev_epoch { + warn!( + mce = self.uploader.max_committed_epoch(), + prev_epoch, "mce higher than clear prev_epoch" + ); + } + for (epoch, result_sender) in self.pending_sync_requests.extract_if(|_, _| true) { send_sync_result( result_sender, @@ -386,19 +482,26 @@ impl HummockEventHandler { ); } - { - Self::for_each_read_version(&self.read_version_mapping, |read_version| { - read_version.clear_uncommitted() - }); - } + assert!( + self.local_read_version_mapping.is_empty(), + "read version mapping not empty when clear. remaining tables: {:?}", + self.local_read_version_mapping + .values() + .map(|read_version| read_version.read().table_id()) + .collect_vec() + ); - self.sstable_object_id_manager - .remove_watermark_object_id(TrackerId::Epoch(HummockEpoch::MAX)); + if let Some(sstable_object_id_manager) = &self.sstable_object_id_manager { + sstable_object_id_manager + .remove_watermark_object_id(TrackerId::Epoch(HummockEpoch::MAX)); + } // Notify completion of the Clear event. let _ = notifier.send(()).inspect_err(|e| { error!("failed to notify completion of clear event: {:?}", e); }); + + info!(prev_epoch, "clear finished"); } fn handle_version_update(&mut self, version_payload: HummockVersionUpdate) { @@ -406,17 +509,34 @@ impl HummockEventHandler { .refiller .last_new_pinned_version() .cloned() - .map(Arc::new) - .unwrap_or_else(|| self.pinned_version.load().clone()); + .unwrap_or_else(|| self.uploader.hummock_version().clone()); let mut sst_delta_infos = vec![]; + let new_pinned_version = Self::resolve_version_update_info( + pinned_version.clone(), + version_payload, + Some(&mut sst_delta_infos), + ); + + self.refiller + .start_cache_refill(sst_delta_infos, pinned_version, new_pinned_version); + } + + fn resolve_version_update_info( + pinned_version: PinnedVersion, + version_payload: HummockVersionUpdate, + mut sst_delta_infos: Option<&mut Vec>, + ) -> PinnedVersion { let newly_pinned_version = match version_payload { HummockVersionUpdate::VersionDeltas(version_deltas) => { let mut version_to_apply = pinned_version.version().clone(); for version_delta in &version_deltas { assert_eq!(version_to_apply.id, version_delta.prev_id); if version_to_apply.max_committed_epoch == version_delta.max_committed_epoch { - sst_delta_infos = version_to_apply.build_sst_delta_infos(version_delta); + if let Some(sst_delta_infos) = &mut sst_delta_infos { + **sst_delta_infos = + version_to_apply.build_sst_delta_infos(version_delta); + } } version_to_apply.apply_version_delta(version_delta); } @@ -428,22 +548,19 @@ impl HummockEventHandler { validate_table_key_range(&newly_pinned_version); - let new_pinned_version = pinned_version.new_pin_version(newly_pinned_version); - - self.refiller - .start_cache_refill(sst_delta_infos, pinned_version, new_pinned_version); + pinned_version.new_pin_version(newly_pinned_version) } fn apply_version_update( &mut self, - pinned_version: Arc, + pinned_version: PinnedVersion, new_pinned_version: PinnedVersion, ) { self.pinned_version .store(Arc::new(new_pinned_version.clone())); { - Self::for_each_read_version(&self.read_version_mapping, |read_version| { + self.for_each_read_version(|read_version| { read_version.update(VersionUpdate::CommittedSnapshot(new_pinned_version.clone())) }); } @@ -465,10 +582,12 @@ impl HummockEventHandler { if let Some(conflict_detector) = self.write_conflict_detector.as_ref() { conflict_detector.set_watermark(max_committed_epoch); } - self.sstable_object_id_manager - .remove_watermark_object_id(TrackerId::Epoch( + + if let Some(sstable_object_id_manager) = &self.sstable_object_id_manager { + sstable_object_id_manager.remove_watermark_object_id(TrackerId::Epoch( self.pinned_version.load().max_committed_epoch(), )); + } debug!( "update to hummock version: {}, epoch: {}", @@ -493,10 +612,26 @@ impl HummockEventHandler { } event = pin!(self.hummock_event_rx.recv()) => { let Some(event) = event else { break }; - if self.handle_hummock_event(event) { - break; + match event { + HummockEvent::Clear(notifier, prev_epoch) => { + self.handle_clear(notifier, prev_epoch).await + }, + HummockEvent::Shutdown => { + info!("event handler shutdown"); + return; + }, + event => { + self.handle_hummock_event(event); + } } } + version_update = pin!(self.version_update_rx.recv()) => { + let Some(version_update) = version_update else { + warn!("version update stream ends. event handle shutdown"); + return; + }; + self.handle_version_update(version_update); + } } } } @@ -513,20 +648,20 @@ impl HummockEventHandler { UploaderEvent::ImmMerged(merge_output) => { // update read version for corresponding table shards - let read_guard = self.read_version_mapping.read(); - if let Some(shards) = read_guard.get(&merge_output.table_id) { - shards.get(&merge_output.instance_id).map_or_else( - || { - warn!( - "handle ImmMerged: table instance not found. table {}, instance {}", - &merge_output.table_id, &merge_output.instance_id - ) - }, - |read_version| { - read_version.write().update(VersionUpdate::Staging( - StagingData::MergedImmMem(merge_output.merged_imm), - )); - }, + if let Some(read_version) = self + .local_read_version_mapping + .get(&merge_output.instance_id) + { + read_version + .write() + .update(VersionUpdate::Staging(StagingData::MergedImmMem( + merge_output.merged_imm, + merge_output.imm_ids, + ))); + } else { + warn!( + "handle ImmMerged: table instance not found. table {:?}, instance {}", + &merge_output.table_id, &merge_output.instance_id ) } } @@ -534,7 +669,7 @@ impl HummockEventHandler { } /// Gracefully shutdown if returns `true`. - fn handle_hummock_event(&mut self, event: HummockEvent) -> bool { + fn handle_hummock_event(&mut self, event: HummockEvent) { match event { HummockEvent::BufferMayFlush => { self.uploader.may_flush(); @@ -545,19 +680,20 @@ impl HummockEventHandler { } => { self.handle_await_sync_epoch(new_sync_epoch, sync_result_sender); } - HummockEvent::Clear(notifier) => { - self.handle_clear(notifier); + HummockEvent::Clear(_, _) => { + unreachable!("clear is handled in separated async context") } HummockEvent::Shutdown => { - info!("buffer tracker shutdown"); - return true; + unreachable!("shutdown is handled specially") } - - HummockEvent::VersionUpdate(version_payload) => { - self.handle_version_update(version_payload); - } - HummockEvent::ImmToUploader(imm) => { + assert!( + self.local_read_version_mapping + .contains_key(&imm.instance_id), + "add imm from non-existing read version instance: instance_id: {}, table_id {}", + imm.instance_id, + imm.table_id, + ); self.uploader.add_imm(imm); self.uploader.may_flush(); } @@ -580,8 +716,14 @@ impl HummockEventHandler { epoch, opts, table_id, - .. + instance_id, } => { + assert!( + self.local_read_version_mapping + .contains_key(&instance_id), + "seal epoch from non-existing read version instance: instance_id: {}, table_id: {}, epoch: {}", + instance_id, table_id, epoch, + ); if let Some((direction, watermarks)) = opts.table_watermarks { self.uploader .add_table_watermarks(epoch, table_id, watermarks, direction) @@ -617,6 +759,8 @@ impl HummockEventHandler { ); { + self.local_read_version_mapping + .insert(instance_id, basic_read_version.clone()); let mut read_version_mapping_guard = self.read_version_mapping.write(); read_version_mapping_guard @@ -651,6 +795,14 @@ impl HummockEventHandler { "read version deregister: table_id: {}, instance_id: {}", table_id, instance_id ); + self.local_read_version_mapping + .remove(&instance_id) + .unwrap_or_else(|| { + panic!( + "DestroyHummockInstance inexist instance table_id {} instance_id {}", + table_id, instance_id + ) + }); let mut read_version_mapping_guard = self.read_version_mapping.write(); let entry = read_version_mapping_guard .get_mut(&table_id) @@ -671,7 +823,6 @@ impl HummockEventHandler { } } } - false } fn generate_instance_id(&mut self) -> LocalInstanceId { @@ -707,6 +858,336 @@ fn to_sync_result(result: &HummockResult) -> HummockResult Err(HummockError::other(format!("sync task failed for {:?}", e))), + Err(e) => Err(HummockError::other(format!( + "sync task failed: {}", + e.as_report() + ))), + } +} + +#[cfg(test)] +mod tests { + use std::future::{poll_fn, Future}; + use std::iter::once; + use std::sync::Arc; + use std::task::Poll; + + use bytes::Bytes; + use futures::FutureExt; + use itertools::Itertools; + use risingwave_common::catalog::TableId; + use risingwave_common::util::epoch::{test_epoch, EpochExt}; + use risingwave_common::util::iter_util::ZipEqDebug; + use risingwave_hummock_sdk::key::TableKey; + use risingwave_hummock_sdk::version::HummockVersion; + use risingwave_pb::hummock::PbHummockVersion; + use tokio::spawn; + use tokio::sync::mpsc::unbounded_channel; + use tokio::sync::oneshot; + use tokio::task::yield_now; + + use crate::hummock::event_handler::refiller::CacheRefiller; + use crate::hummock::event_handler::{HummockEvent, HummockEventHandler, HummockVersionUpdate}; + use crate::hummock::iterator::test_utils::mock_sstable_store; + use crate::hummock::local_version::pinned_version::PinnedVersion; + use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch; + use crate::hummock::store::version::{StagingData, VersionUpdate}; + use crate::hummock::test_utils::default_opts_for_test; + use crate::hummock::value::HummockValue; + use crate::hummock::HummockError; + use crate::monitor::HummockStateStoreMetrics; + + #[tokio::test] + async fn test_event_handler_merging_task() { + let table_id = TableId::new(123); + let epoch0 = test_epoch(233); + let pinned_version = PinnedVersion::new( + HummockVersion::from_rpc_protobuf(&PbHummockVersion { + id: 1, + max_committed_epoch: epoch0, + ..Default::default() + }), + unbounded_channel().0, + ); + + let mut storage_opts = default_opts_for_test(); + storage_opts.imm_merge_threshold = 5; + + let (_version_update_tx, version_update_rx) = unbounded_channel(); + + let (spawn_upload_task_tx, mut spawn_upload_task_rx) = unbounded_channel(); + let (spawn_merging_task_tx, mut spawn_merging_task_rx) = unbounded_channel(); + let event_handler = HummockEventHandler::new_inner( + version_update_rx, + pinned_version, + None, + mock_sstable_store(), + Arc::new(HummockStateStoreMetrics::unused()), + &storage_opts, + Arc::new(move |_, _| { + let (tx, rx) = oneshot::channel::<()>(); + spawn_upload_task_tx.send(tx).unwrap(); + spawn(async move { + // wait for main thread to notify returning error + rx.await.unwrap(); + Err(HummockError::other("".to_string())) + }) + }), + Arc::new(move |_, _, imms, _| { + let (tx, rx) = oneshot::channel::<()>(); + let (finish_tx, finish_rx) = oneshot::channel::<()>(); + spawn_merging_task_tx.send((tx, finish_rx)).unwrap(); + spawn(async move { + rx.await.unwrap(); + finish_tx.send(()).unwrap(); + imms[0].clone() + }) + }), + CacheRefiller::default_spawn_refill_task(), + ); + + let tx = event_handler.event_sender(); + + let _join_handle = spawn(event_handler.start_hummock_event_handler_worker()); + + let (read_version_tx, read_version_rx) = oneshot::channel(); + + tx.send(HummockEvent::RegisterReadVersion { + table_id, + new_read_version_sender: read_version_tx, + is_replicated: false, + }) + .unwrap(); + let (read_version, guard) = read_version_rx.await.unwrap(); + let instance_id = guard.instance_id; + + let build_batch = |epoch, spill_offset| { + SharedBufferBatch::build_shared_buffer_batch( + epoch, + spill_offset, + vec![(TableKey(Bytes::from("key")), HummockValue::Delete)], + 10, + table_id, + instance_id, + None, + ) + }; + + let epoch1 = epoch0.next_epoch(); + let imm1 = build_batch(epoch1, 0); + read_version + .write() + .update(VersionUpdate::Staging(StagingData::ImmMem(imm1.clone()))); + tx.send(HummockEvent::ImmToUploader(imm1.clone())).unwrap(); + tx.send(HummockEvent::SealEpoch { + epoch: epoch1, + is_checkpoint: true, + }) + .unwrap(); + let (sync_tx, mut sync_rx) = oneshot::channel(); + tx.send(HummockEvent::AwaitSyncEpoch { + new_sync_epoch: epoch1, + sync_result_sender: sync_tx, + }) + .unwrap(); + + let upload_finish_tx = spawn_upload_task_rx.recv().await.unwrap(); + assert!(poll_fn(|cx| Poll::Ready(sync_rx.poll_unpin(cx))) + .await + .is_pending()); + + let epoch2 = epoch1.next_epoch(); + let mut imm_ids = Vec::new(); + for i in 0..10 { + let imm = build_batch(epoch2, i); + imm_ids.push(imm.batch_id()); + read_version + .write() + .update(VersionUpdate::Staging(StagingData::ImmMem(imm.clone()))); + tx.send(HummockEvent::ImmToUploader(imm)).unwrap(); + } + + for (staging_imm, imm_id) in read_version + .read() + .staging() + .imm + .iter() + .zip_eq_debug(imm_ids.iter().copied().rev().chain(once(imm1.batch_id()))) + { + assert_eq!(staging_imm.batch_id(), imm_id); + } + + // should start merging task + tx.send(HummockEvent::SealEpoch { + epoch: epoch2, + is_checkpoint: false, + }) + .unwrap(); + + println!("before wait spawn merging task"); + + let (merging_start_tx, merging_finish_rx) = spawn_merging_task_rx.recv().await.unwrap(); + merging_start_tx.send(()).unwrap(); + + println!("after wait spawn merging task"); + + // yield to possibly poll the merging task, though it shouldn't poll it because there is unfinished syncing task + yield_now().await; + + for (staging_imm, imm_id) in read_version + .read() + .staging() + .imm + .iter() + .zip_eq_debug(imm_ids.iter().copied().rev().chain(once(imm1.batch_id()))) + { + assert_eq!(staging_imm.batch_id(), imm_id); + } + + upload_finish_tx.send(()).unwrap(); + assert!(sync_rx.await.unwrap().is_err()); + + merging_finish_rx.await.unwrap(); + + // yield to poll the merging task, and then it should have finished. + for _ in 0..10 { + yield_now().await; + } + + assert_eq!( + read_version + .read() + .staging() + .imm + .iter() + .map(|imm| imm.batch_id()) + .collect_vec(), + vec![*imm_ids.last().unwrap(), imm1.batch_id()] + ); + } + + #[tokio::test] + async fn test_clear_shared_buffer() { + let epoch0 = 233; + let mut next_version_id = 1; + let mut make_new_version = |max_committed_epoch| { + let id = next_version_id; + next_version_id += 1; + HummockVersion::from_rpc_protobuf(&PbHummockVersion { + id, + max_committed_epoch, + ..Default::default() + }) + }; + + let initial_version = PinnedVersion::new(make_new_version(epoch0), unbounded_channel().0); + + let (version_update_tx, version_update_rx) = unbounded_channel(); + let (refill_task_tx, mut refill_task_rx) = unbounded_channel(); + + let refill_task_tx_clone = refill_task_tx.clone(); + + let event_handler = HummockEventHandler::new_inner( + version_update_rx, + initial_version.clone(), + None, + mock_sstable_store(), + Arc::new(HummockStateStoreMetrics::unused()), + &default_opts_for_test(), + Arc::new(|_, _| unreachable!("should not spawn upload task")), + Arc::new(|_, _, _, _| unreachable!("should not spawn merging task")), + Arc::new(move |_, _, old_version, new_version| { + let (tx, rx) = oneshot::channel(); + refill_task_tx_clone + .send((old_version, new_version, tx)) + .unwrap(); + spawn(async move { + let _ = rx.await; + }) + }), + ); + + let event_tx = event_handler.event_sender(); + let latest_version = event_handler.pinned_version.clone(); + let latest_version_update_tx = event_handler.version_update_notifier_tx.clone(); + + let send_clear = |epoch| { + let (tx, rx) = oneshot::channel(); + event_tx.send(HummockEvent::Clear(tx, epoch)).unwrap(); + rx + }; + + let _join_handle = spawn(event_handler.start_hummock_event_handler_worker()); + + // test normal recovery + send_clear(epoch0).await.unwrap(); + + // test normal refill finish + let epoch1 = epoch0 + 1; + let version1 = make_new_version(epoch1); + { + version_update_tx + .send(HummockVersionUpdate::PinnedVersion(version1.clone())) + .unwrap(); + let (old_version, new_version, refill_finish_tx) = refill_task_rx.recv().await.unwrap(); + assert_eq!(old_version.version(), initial_version.version()); + assert_eq!(new_version.version(), &version1); + assert_eq!(latest_version.load().version(), initial_version.version()); + + let mut changed = latest_version_update_tx.subscribe(); + refill_finish_tx.send(()).unwrap(); + changed.changed().await.unwrap(); + assert_eq!(latest_version.load().version(), &version1); + } + + // test recovery with pending refill task + let epoch2 = epoch1 + 1; + let version2 = make_new_version(epoch2); + let epoch3 = epoch2 + 1; + let version3 = make_new_version(epoch3); + { + version_update_tx + .send(HummockVersionUpdate::PinnedVersion(version2.clone())) + .unwrap(); + version_update_tx + .send(HummockVersionUpdate::PinnedVersion(version3.clone())) + .unwrap(); + let (old_version2, new_version2, _refill_finish_tx2) = + refill_task_rx.recv().await.unwrap(); + assert_eq!(old_version2.version(), &version1); + assert_eq!(new_version2.version(), &version2); + let (old_version3, new_version3, _refill_finish_tx3) = + refill_task_rx.recv().await.unwrap(); + assert_eq!(old_version3.version(), &version2); + assert_eq!(new_version3.version(), &version3); + assert_eq!(latest_version.load().version(), &version1); + + let rx = send_clear(epoch3); + rx.await.unwrap(); + assert_eq!(latest_version.load().version(), &version3); + } + + async fn assert_pending(fut: &mut (impl Future + Unpin)) { + assert!(poll_fn(|cx| Poll::Ready(fut.poll_unpin(cx).is_pending())).await); + } + + // test recovery with later arriving version update + let epoch4 = epoch3 + 1; + let version4 = make_new_version(epoch4); + let epoch5 = epoch4 + 1; + let version5 = make_new_version(epoch5); + { + let mut rx = send_clear(epoch5); + assert_pending(&mut rx).await; + version_update_tx + .send(HummockVersionUpdate::PinnedVersion(version4.clone())) + .unwrap(); + assert_pending(&mut rx).await; + version_update_tx + .send(HummockVersionUpdate::PinnedVersion(version5.clone())) + .unwrap(); + rx.await.unwrap(); + assert_eq!(latest_version.load().version(), &version5); + } } } diff --git a/src/storage/src/hummock/event_handler/mod.rs b/src/storage/src/hummock/event_handler/mod.rs index 42b636fa8a1bd..5c2e081c19f72 100644 --- a/src/storage/src/hummock/event_handler/mod.rs +++ b/src/storage/src/hummock/event_handler/mod.rs @@ -15,9 +15,10 @@ use std::collections::HashMap; use std::sync::Arc; -use parking_lot::RwLock; +use parking_lot::{RwLock, RwLockReadGuard}; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::HummockEpoch; +use thiserror_ext::AsReport; use tokio::sync::{mpsc, oneshot}; use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatch; @@ -60,12 +61,10 @@ pub enum HummockEvent { }, /// Clear shared buffer and reset all states - Clear(oneshot::Sender<()>), + Clear(oneshot::Sender<()>, u64), Shutdown, - VersionUpdate(HummockVersionUpdate), - ImmToUploader(ImmutableMemtable), SealEpoch { @@ -87,8 +86,7 @@ pub enum HummockEvent { RegisterReadVersion { table_id: TableId, - new_read_version_sender: - oneshot::Sender<(Arc>, LocalInstanceGuard)>, + new_read_version_sender: oneshot::Sender<(HummockReadVersionRef, LocalInstanceGuard)>, is_replicated: bool, }, @@ -108,14 +106,10 @@ impl HummockEvent { sync_result_sender: _, } => format!("AwaitSyncEpoch epoch {} ", new_sync_epoch), - HummockEvent::Clear(_) => "Clear".to_string(), + HummockEvent::Clear(_, prev_epoch) => format!("Clear {:?}", prev_epoch), HummockEvent::Shutdown => "Shutdown".to_string(), - HummockEvent::VersionUpdate(version_update_payload) => { - format!("VersionUpdate {:?}", version_update_payload) - } - HummockEvent::ImmToUploader(imm) => { format!("ImmToUploader {:?}", imm) } @@ -172,8 +166,27 @@ impl std::fmt::Debug for HummockEvent { } pub type LocalInstanceId = u64; -pub type ReadVersionMappingType = - RwLock>>>>; +pub type HummockReadVersionRef = Arc>; +pub type ReadVersionMappingType = HashMap>; +pub type ReadOnlyReadVersionMapping = ReadOnlyRwLockRef; + +pub struct ReadOnlyRwLockRef(Arc>); + +impl Clone for ReadOnlyRwLockRef { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl ReadOnlyRwLockRef { + pub fn new(inner: Arc>) -> Self { + Self(inner) + } + + pub fn read(&self) -> RwLockReadGuard<'_, T> { + self.0.read() + } +} pub struct LocalInstanceGuard { pub table_id: TableId, @@ -192,10 +205,10 @@ impl Drop for LocalInstanceGuard { }) .unwrap_or_else(|err| { tracing::error!( - "LocalInstanceGuard table_id {:?} instance_id {} Drop SendError {:?}", - self.table_id, - self.instance_id, - err + error = %err.as_report(), + table_id = %self.table_id, + instance_id = self.instance_id, + "LocalInstanceGuard Drop SendError", ) }) } diff --git a/src/storage/src/hummock/event_handler/refiller.rs b/src/storage/src/hummock/event_handler/refiller.rs index d124d067ac67d..1c592fac85a2d 100644 --- a/src/storage/src/hummock/event_handler/refiller.rs +++ b/src/storage/src/hummock/event_handler/refiller.rs @@ -13,10 +13,10 @@ // limitations under the License. use std::collections::{HashMap, HashSet, VecDeque}; -use std::ops::{Deref, DerefMut, Range}; -use std::pin::Pin; +use std::future::poll_fn; +use std::ops::{Deref, Range}; use std::sync::{Arc, LazyLock}; -use std::task::{ready, Context, Poll}; +use std::task::{ready, Poll}; use std::time::{Duration, Instant}; use foyer::common::code::Key; @@ -33,6 +33,7 @@ use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_hummock_sdk::compaction_group::hummock_version_ext::SstDeltaInfo; use risingwave_hummock_sdk::{HummockSstableObjectId, KeyComparator}; +use thiserror_ext::AsReport; use tokio::sync::Semaphore; use tokio::task::JoinHandle; @@ -43,6 +44,7 @@ use crate::hummock::{ SstableStoreRef, TableHolder, }; use crate::monitor::StoreLocalStatistic; +use crate::opts::StorageOpts; pub static GLOBAL_CACHE_REFILL_METRICS: LazyLock = LazyLock::new(|| CacheRefillMetrics::new(&GLOBAL_METRICS_REGISTRY)); @@ -200,21 +202,51 @@ pub struct CacheRefillConfig { pub threshold: f64, } +impl CacheRefillConfig { + pub fn from_storage_opts(options: &StorageOpts) -> Self { + Self { + timeout: Duration::from_millis(options.cache_refill_timeout_ms), + data_refill_levels: options + .cache_refill_data_refill_levels + .iter() + .copied() + .collect(), + concurrency: options.cache_refill_concurrency, + unit: options.cache_refill_unit, + threshold: options.cache_refill_threshold, + } + } +} + struct Item { handle: JoinHandle<()>, event: CacheRefillerEvent, } +pub(crate) type SpawnRefillTask = Arc< + // first current version, second new version + dyn Fn(Vec, CacheRefillContext, PinnedVersion, PinnedVersion) -> JoinHandle<()> + + Send + + Sync + + 'static, +>; + /// A cache refiller for hummock data. -pub struct CacheRefiller { +pub(crate) struct CacheRefiller { /// order: old => new queue: VecDeque, context: CacheRefillContext, + + spawn_refill_task: SpawnRefillTask, } impl CacheRefiller { - pub fn new(config: CacheRefillConfig, sstable_store: SstableStoreRef) -> Self { + pub(crate) fn new( + config: CacheRefillConfig, + sstable_store: SstableStoreRef, + spawn_refill_task: SpawnRefillTask, + ) -> Self { let config = Arc::new(config); let concurrency = Arc::new(Semaphore::new(config.concurrency)); Self { @@ -224,71 +256,87 @@ impl CacheRefiller { concurrency, sstable_store, }, + spawn_refill_task, } } - pub fn start_cache_refill( + pub(crate) fn default_spawn_refill_task() -> SpawnRefillTask { + Arc::new(|deltas, context, _, _| { + let task = CacheRefillTask { deltas, context }; + tokio::spawn(task.run()) + }) + } + + pub(crate) fn start_cache_refill( &mut self, deltas: Vec, - pinned_version: Arc, + pinned_version: PinnedVersion, new_pinned_version: PinnedVersion, ) { - let task = CacheRefillTask { + let handle = (self.spawn_refill_task)( deltas, - context: self.context.clone(), - }; + self.context.clone(), + pinned_version.clone(), + new_pinned_version.clone(), + ); let event = CacheRefillerEvent { pinned_version, new_pinned_version, }; - let handle = tokio::spawn(task.run()); let item = Item { handle, event }; self.queue.push_back(item); GLOBAL_CACHE_REFILL_METRICS.refill_queue_total.add(1); } - pub fn last_new_pinned_version(&self) -> Option<&PinnedVersion> { + pub(crate) fn last_new_pinned_version(&self) -> Option<&PinnedVersion> { self.queue.back().map(|item| &item.event.new_pinned_version) } - pub fn next_event(&mut self) -> NextCacheRefillerEvent<'_> { - NextCacheRefillerEvent { refiller: self } + /// Clear the queue for cache refill and return an event that merges all pending cache refill events + /// into a single event that takes the earliest and latest version. + pub(crate) fn clear(&mut self) -> Option { + let Some(last_item) = self.queue.pop_back() else { + return None; + }; + let mut event = last_item.event; + while let Some(item) = self.queue.pop_back() { + assert_eq!( + event.pinned_version.id(), + item.event.new_pinned_version.id() + ); + event.pinned_version = item.event.pinned_version; + } + Some(event) } } -pub struct NextCacheRefillerEvent<'a> { - refiller: &'a mut CacheRefiller, -} - -impl<'a> Future for NextCacheRefillerEvent<'a> { - type Output = CacheRefillerEvent; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let refiller = &mut self.deref_mut().refiller; - - if let Some(item) = refiller.queue.front_mut() { - ready!(item.handle.poll_unpin(cx)).unwrap(); - let item = refiller.queue.pop_front().unwrap(); - GLOBAL_CACHE_REFILL_METRICS.refill_queue_total.sub(1); - return Poll::Ready(item.event); - } - Poll::Pending +impl CacheRefiller { + pub(crate) fn next_event(&mut self) -> impl Future + '_ { + poll_fn(|cx| { + if let Some(item) = self.queue.front_mut() { + ready!(item.handle.poll_unpin(cx)).unwrap(); + let item = self.queue.pop_front().unwrap(); + GLOBAL_CACHE_REFILL_METRICS.refill_queue_total.sub(1); + return Poll::Ready(item.event); + } + Poll::Pending + }) } } pub struct CacheRefillerEvent { - pub pinned_version: Arc, + pub pinned_version: PinnedVersion, pub new_pinned_version: PinnedVersion, } #[derive(Clone)] -struct CacheRefillContext { +pub(crate) struct CacheRefillContext { config: Arc, concurrency: Arc, sstable_store: SstableStoreRef, } -pub struct CacheRefillTask { +struct CacheRefillTask { deltas: Vec, context: CacheRefillContext, } @@ -304,7 +352,7 @@ impl CacheRefillTask { let holders = match Self::meta_cache_refill(&context, delta).await { Ok(holders) => holders, Err(e) => { - tracing::warn!("meta cache refill error: {:?}", e); + tracing::warn!(error = %e.as_report(), "meta cache refill error"); return; } }; @@ -514,7 +562,9 @@ impl CacheRefillTask { }); let parent_ssts = match try_join_all(futures).await { Ok(parent_ssts) => parent_ssts.into_iter().flatten().collect_vec(), - Err(e) => return tracing::error!("get old meta from cache error: {}", e), + Err(e) => { + return tracing::error!(error = %e.as_report(), "get old meta from cache error") + } }; let units = Self::get_units_to_refill_by_inheritance(context, &holders, &parent_ssts); @@ -525,7 +575,7 @@ impl CacheRefillTask { async move { let sst = ssts.get(&unit.sst_obj_id).unwrap(); if let Err(e) = Self::data_file_cache_refill_unit(context, sst, unit).await { - tracing::error!("data file cache unit refill error: {}", e); + tracing::error!(error = %e.as_report(), "data file cache unit refill error"); } } }); diff --git a/src/storage/src/hummock/event_handler/uploader.rs b/src/storage/src/hummock/event_handler/uploader.rs index 067200d699ec6..0f9f6132472a7 100644 --- a/src/storage/src/hummock/event_handler/uploader.rs +++ b/src/storage/src/hummock/event_handler/uploader.rs @@ -15,9 +15,8 @@ use std::collections::hash_map::Entry; use std::collections::{BTreeMap, HashMap, VecDeque}; use std::fmt::{Debug, Display, Formatter}; -use std::future::Future; +use std::future::{poll_fn, Future}; use std::mem::swap; -use std::ops::DerefMut; use std::pin::Pin; use std::sync::Arc; use std::task::{ready, Context, Poll}; @@ -35,6 +34,7 @@ use risingwave_hummock_sdk::table_watermark::{ TableWatermarks, VnodeWatermark, WatermarkDirection, }; use risingwave_hummock_sdk::{CompactionGroupId, HummockEpoch, LocalSstableInfo}; +use thiserror_ext::AsReport; use tokio::task::JoinHandle; use tracing::{debug, error, info}; @@ -59,6 +59,26 @@ pub type SpawnUploadTask = Arc< + 'static, >; +pub type SpawnMergingTask = Arc< + dyn Fn( + TableId, + LocalInstanceId, + Vec, + Option, + ) -> JoinHandle + + Send + + Sync + + 'static, +>; + +pub(crate) fn default_spawn_merging_task( + compaction_executor: Arc, +) -> SpawnMergingTask { + Arc::new(move |table_id, instance_id, imms, tracker| { + compaction_executor.spawn(merge_imms_in_memory(table_id, instance_id, imms, tracker)) + }) +} + #[derive(Clone)] pub struct UploadTaskInfo { pub task_size: usize, @@ -98,6 +118,8 @@ struct UploadingTask { } pub struct MergeImmTaskOutput { + /// Input imm ids of the merging task. Larger imm ids at the front. + pub imm_ids: Vec, pub table_id: TableId, pub instance_id: LocalInstanceId, pub merged_imm: ImmutableMemtable, @@ -108,7 +130,17 @@ struct MergingImmTask { table_id: TableId, instance_id: LocalInstanceId, input_imms: Vec, - join_handle: JoinHandle>, + join_handle: JoinHandle, +} + +impl Debug for MergingImmTask { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("MergingImmTask") + .field("table_id", &self.table_id) + .field("instance_id", &self.instance_id) + .field("input_imms", &self.input_imms) + .finish() + } } impl MergingImmTask { @@ -119,10 +151,9 @@ impl MergingImmTask { memory_tracker: Option, context: &UploaderContext, ) -> Self { + assert!(imms.iter().rev().map(|imm| imm.batch_id()).is_sorted()); let input_imms = imms.clone(); - let join_handle = context.compaction_executor.spawn(async move { - merge_imms_in_memory(table_id, instance_id, imms, memory_tracker).await - }); + let join_handle = (context.spawn_merging_task)(table_id, instance_id, imms, memory_tracker); MergingImmTask { table_id, @@ -133,19 +164,22 @@ impl MergingImmTask { } /// Poll the result of the merge task - fn poll_result(&mut self, cx: &mut Context<'_>) -> Poll> { + fn poll_result(&mut self, cx: &mut Context<'_>) -> Poll { Poll::Ready(match ready!(self.join_handle.poll_unpin(cx)) { Ok(task_result) => task_result, - Err(err) => Err(HummockError::other(format!( - "fail to join imm merge join handle: {:?}", - err - ))), + Err(err) => { + panic!( + "failed to join merging task: {:?} {:?}", + err.as_report(), + self + ); + } }) } } impl Future for MergingImmTask { - type Output = HummockResult; + type Output = ImmutableMemtable; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { self.poll_result(cx) @@ -215,11 +249,12 @@ impl UploadingTask { Ok(task_result) => task_result .inspect(|_| { if self.task_info.task_size > Self::LOG_THRESHOLD_FOR_UPLOAD_TASK_SIZE { - info!("upload task finish {:?}", self.task_info) + info!(task_info = ?self.task_info, "upload task finish"); } else { - debug!("upload task finish {:?}", self.task_info) + debug!(task_info = ?self.task_info, "upload task finish"); } }) + .inspect_err(|e| error!(task_info = ?self.task_info, err = ?e.as_report(), "upload task failed")) .map(|ssts| { StagingSstableInfo::new( ssts, @@ -230,8 +265,8 @@ impl UploadingTask { }), Err(err) => Err(HummockError::other(format!( - "fail to join upload join handle: {:?}", - err + "fail to join upload join handle: {}", + err.as_report() ))), }) } @@ -244,8 +279,9 @@ impl UploadingTask { Ok(sstables) => return Poll::Ready(sstables), Err(e) => { error!( - "a flush task {:?} failed, start retry. Task info: {:?}", - self.task_info, e + error = %e.as_report(), + task_info = ?self.task_info, + "a flush task failed, start retry", ); self.join_handle = (self.spawn_upload_task)(self.payload.clone(), self.task_info.clone()); @@ -389,6 +425,7 @@ struct SealedData { merged_imms: VecDeque, // Sealed imms grouped by table shard. + // newer data (larger imm id) at the front imms_by_table_shard: HashMap<(TableId, LocalInstanceId), VecDeque>, // Merging tasks generated from sealed imms @@ -452,10 +489,14 @@ impl SealedData { // rearrange sealed imms by table shard and in epoch descending order for imm in unseal_epoch_data.imms.into_iter().rev() { - self.imms_by_table_shard + let queue = self + .imms_by_table_shard .entry((imm.table_id, imm.instance_id)) - .or_default() - .push_front(imm); + .or_default(); + if let Some(front) = queue.front() { + assert_gt!(imm.batch_id(), front.batch_id()); + } + queue.push_front(imm); } self.epochs.push_front(epoch); @@ -484,7 +525,6 @@ impl SealedData { } fn add_merged_imm(&mut self, merged_imm: &ImmutableMemtable) { - debug_assert!(merged_imm.is_merged_imm()); // add merged_imm to merged_imms self.merged_imms.push_front(merged_imm.clone()); } @@ -547,15 +587,16 @@ impl SealedData { fn poll_success_merge_imm(&mut self, cx: &mut Context<'_>) -> Poll> { // only poll the oldest merge task if there is any if let Some(task) = self.merging_tasks.back_mut() { - let merge_result = ready!(task.poll_unpin(cx)); + let merged_imm = ready!(task.poll_unpin(cx)); // pop the finished task let task = self.merging_tasks.pop_back().expect("must exist"); Poll::Ready(Some(MergeImmTaskOutput { + imm_ids: task.input_imms.iter().map(|imm| imm.batch_id()).collect(), table_id: task.table_id, instance_id: task.instance_id, - merged_imm: merge_result.unwrap(), + merged_imm, })) } else { Poll::Ready(None) @@ -618,14 +659,13 @@ struct UploaderContext { pinned_version: PinnedVersion, /// When called, it will spawn a task to flush the imm into sst and return the join handle. spawn_upload_task: SpawnUploadTask, + spawn_merging_task: SpawnMergingTask, buffer_tracker: BufferTracker, /// The number of immutable memtables that will be merged into a new imm. /// When the number of imms of a table shard exceeds this threshold, uploader will generate /// merging tasks to merge them. imm_merge_threshold: usize, - compaction_executor: Arc, - stats: Arc, } @@ -633,17 +673,17 @@ impl UploaderContext { fn new( pinned_version: PinnedVersion, spawn_upload_task: SpawnUploadTask, + spawn_merging_task: SpawnMergingTask, buffer_tracker: BufferTracker, config: &StorageOpts, - compaction_executor: Arc, stats: Arc, ) -> Self { UploaderContext { pinned_version, spawn_upload_task, + spawn_merging_task, buffer_tracker, imm_merge_threshold: config.imm_merge_threshold, - compaction_executor, stats, } } @@ -694,9 +734,9 @@ impl HummockUploader { state_store_metrics: Arc, pinned_version: PinnedVersion, spawn_upload_task: SpawnUploadTask, + spawn_merging_task: SpawnMergingTask, buffer_tracker: BufferTracker, config: &StorageOpts, - compaction_executor: Arc, ) -> Self { let initial_epoch = pinned_version.version().max_committed_epoch; Self { @@ -710,9 +750,9 @@ impl HummockUploader { context: UploaderContext::new( pinned_version, spawn_upload_task, + spawn_merging_task, buffer_tracker, config, - compaction_executor, state_store_metrics, ), } @@ -739,6 +779,10 @@ impl HummockUploader { self.context.pinned_version.max_committed_epoch() } + pub(crate) fn hummock_version(&self) -> &PinnedVersion { + &self.context.pinned_version + } + pub(crate) fn get_synced_data(&self, epoch: HummockEpoch) -> Option<&SyncedDataState> { assert!(self.max_committed_epoch() < epoch && epoch <= self.max_synced_epoch); self.synced_data.get(&epoch) @@ -824,18 +868,18 @@ impl HummockUploader { .filter(|(_, imms)| imms.len() >= self.context.imm_merge_threshold) { let imms_to_merge = imms.drain(..).collect_vec(); - let mut kv_count = 0; + let mut value_count = 0; let mut imm_size = 0; imms_to_merge.iter().for_each(|imm| { // ensure imms are sealed assert_le!(imm.max_epoch(), sealed_epoch); - kv_count += imm.kv_count(); + value_count += imm.value_count(); imm_size += imm.size(); }); // acquire memory before generate merge task // if acquire memory failed, the task will not be generated - let memory_sz = (imm_size + kv_count * EPOCH_LEN) as u64; + let memory_sz = (imm_size + value_count * EPOCH_LEN) as u64; if let Some(tracker) = memory_limiter.try_require_memory(memory_sz) { self.sealed_data .merging_tasks @@ -996,10 +1040,6 @@ impl HummockUploader { // TODO: call `abort` on the uploading task join handle } - - pub(crate) fn next_event(&mut self) -> NextUploaderEvent<'_> { - NextUploaderEvent { uploader: self } - } } impl HummockUploader { @@ -1085,16 +1125,14 @@ impl HummockUploader { .stats .merge_imm_batch_memory_sz .with_label_values(&[table_id_label.as_str()]) - .inc_by((output.merged_imm.size() + output.merged_imm.kv_count() * EPOCH_LEN) as _); + .inc_by( + (output.merged_imm.size() + output.merged_imm.value_count() * EPOCH_LEN) as _, + ); } poll_ret } } -pub(crate) struct NextUploaderEvent<'a> { - uploader: &'a mut HummockUploader, -} - pub(crate) enum UploaderEvent { // staging sstable info of newer data comes first SyncFinish(HummockEpoch, Vec), @@ -1102,30 +1140,28 @@ pub(crate) enum UploaderEvent { ImmMerged(MergeImmTaskOutput), } -impl<'a> Future for NextUploaderEvent<'a> { - type Output = UploaderEvent; - - fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - let uploader = &mut self.deref_mut().uploader; - - if let Some((epoch, newly_uploaded_sstables)) = ready!(uploader.poll_syncing_task(cx)) { - return Poll::Ready(UploaderEvent::SyncFinish(epoch, newly_uploaded_sstables)); - } +impl HummockUploader { + pub(crate) fn next_event(&mut self) -> impl Future + '_ { + poll_fn(|cx| { + if let Some((epoch, newly_uploaded_sstables)) = ready!(self.poll_syncing_task(cx)) { + return Poll::Ready(UploaderEvent::SyncFinish(epoch, newly_uploaded_sstables)); + } - if let Some(sstable_info) = ready!(uploader.poll_sealed_spill_task(cx)) { - return Poll::Ready(UploaderEvent::DataSpilled(sstable_info)); - } + if let Some(sstable_info) = ready!(self.poll_sealed_spill_task(cx)) { + return Poll::Ready(UploaderEvent::DataSpilled(sstable_info)); + } - if let Some(sstable_info) = ready!(uploader.poll_unsealed_spill_task(cx)) { - return Poll::Ready(UploaderEvent::DataSpilled(sstable_info)); - } + if let Some(sstable_info) = ready!(self.poll_unsealed_spill_task(cx)) { + return Poll::Ready(UploaderEvent::DataSpilled(sstable_info)); + } - if let Some(merge_output) = ready!(uploader.poll_sealed_merge_imm_task(cx)) { - // add the merged imm into sealed data - uploader.update_sealed_data(&merge_output.merged_imm); - return Poll::Ready(UploaderEvent::ImmMerged(merge_output)); - } - Poll::Pending + if let Some(merge_output) = ready!(self.poll_sealed_merge_imm_task(cx)) { + // add the merged imm into sealed data + self.update_sealed_data(&merge_output.merged_imm); + return Poll::Ready(UploaderEvent::ImmMerged(merge_output)); + } + Poll::Pending + }) } } @@ -1159,8 +1195,8 @@ mod tests { use crate::hummock::compactor::CompactionExecutor; use crate::hummock::event_handler::hummock_event_handler::BufferTracker; use crate::hummock::event_handler::uploader::{ - HummockUploader, MergingImmTask, UploadTaskInfo, UploadTaskOutput, UploadTaskPayload, - UploaderContext, UploaderEvent, UploadingTask, + default_spawn_merging_task, HummockUploader, MergingImmTask, UploadTaskInfo, + UploadTaskOutput, UploadTaskPayload, UploaderContext, UploaderEvent, UploadingTask, }; use crate::hummock::event_handler::LocalInstanceId; use crate::hummock::iterator::test_utils::{ @@ -1219,9 +1255,8 @@ mod tests { 0, sorted_items, size, - vec![], TEST_TABLE_ID, - None, + LocalInstanceId::default(), tracker, ) } @@ -1268,9 +1303,9 @@ mod tests { UploaderContext::new( initial_pinned_version(), Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))), + default_spawn_merging_task(compaction_executor), BufferTracker::for_test(), &config, - compaction_executor, Arc::new(HummockStateStoreMetrics::unused()), ) } @@ -1289,9 +1324,9 @@ mod tests { Arc::new(HummockStateStoreMetrics::unused()), initial_pinned_version(), Arc::new(move |payload, task_info| spawn(upload_fn(payload, task_info))), + default_spawn_merging_task(compaction_executor), BufferTracker::for_test(), &config, - compaction_executor, ) } @@ -1653,17 +1688,10 @@ mod tests { _ = sleep => { println!("sleep timeout") } - res = &mut task => { - match res { - Ok(imm) => { - println!("merging task success"); - assert_eq!(table_id, imm.table_id); - assert_eq!(9, imm.kv_count()); - } - Err(err) => { - println!("merging task failed: {:?}", err); - } - } + imm = &mut task => { + println!("merging task success"); + assert_eq!(table_id, imm.table_id); + assert_eq!(9, imm.value_count()); } } task.join_handle.abort(); @@ -1786,9 +1814,9 @@ mod tests { }) } }), + default_spawn_merging_task(compaction_executor), buffer_tracker.clone(), &config, - compaction_executor, ); (buffer_tracker, uploader, new_task_notifier) } diff --git a/src/storage/src/hummock/file_cache/store.rs b/src/storage/src/hummock/file_cache/store.rs index d2bfecee77229..e47ed830d2199 100644 --- a/src/storage/src/hummock/file_cache/store.rs +++ b/src/storage/src/hummock/file_cache/store.rs @@ -455,7 +455,7 @@ impl Value for CachedBlock { fn serialized_len(&self) -> usize { 1 /* type */ + match self { - CachedBlock::Loaded { block } => block.raw_data().len(), + CachedBlock::Loaded { block } => block.raw().len(), CachedBlock::Fetched { bytes, uncompressed_capacity: _ } => 8 + bytes.len(), } } @@ -506,7 +506,7 @@ impl std::io::Read for CachedBlockCursor { if self.pos < 1 { self.pos += copy([0], &mut buf); } - self.pos += copy(&block.raw_data()[self.pos - 1..], &mut buf); + self.pos += copy(&block.raw()[self.pos - 1..], &mut buf); } CachedBlock::Fetched { bytes, @@ -541,7 +541,7 @@ impl Value for Box { type Cursor = BoxBlockCursor; fn serialized_len(&self) -> usize { - self.raw_data().len() + self.raw().len() } fn read(buf: &[u8]) -> CodingResult { @@ -571,7 +571,7 @@ impl BoxBlockCursor { impl std::io::Read for BoxBlockCursor { fn read(&mut self, buf: &mut [u8]) -> std::io::Result { let pos = self.pos; - self.pos += copy(&self.inner.raw_data()[self.pos..], buf); + self.pos += copy(&self.inner.raw()[self.pos..], buf); let n = self.pos - pos; Ok(n) } @@ -714,13 +714,8 @@ mod tests { b"v04", ); - Box::new( - Block::decode( - builder.build().to_vec().into(), - builder.uncompressed_block_size(), - ) - .unwrap(), - ) + let uncompress = builder.uncompressed_block_size(); + Box::new(Block::decode(builder.build().to_vec().into(), uncompress).unwrap()) } fn sstable_for_test() -> Sstable { @@ -761,7 +756,7 @@ mod tests { std::io::copy(&mut cursor, &mut buf).unwrap(); let target = cursor.into_inner(); let block = Box::::read(&buf[..]).unwrap(); - assert_eq!(target.raw_data(), block.raw_data()); + assert_eq!(target.raw(), block.raw()); } { @@ -792,7 +787,7 @@ mod tests { CachedBlock::Loaded { block } => block, CachedBlock::Fetched { .. } => panic!(), }; - assert_eq!(target.raw_data(), block.raw_data()); + assert_eq!(target.raw(), block.raw()); } { diff --git a/src/storage/src/hummock/iterator/backward_merge.rs b/src/storage/src/hummock/iterator/backward_merge.rs index cdcad9d53dc7d..f8464a777e078 100644 --- a/src/storage/src/hummock/iterator/backward_merge.rs +++ b/src/storage/src/hummock/iterator/backward_merge.rs @@ -18,7 +18,7 @@ mod test { default_builder_opt_for_test, gen_iterator_test_sstable_base, iterator_test_key_of, iterator_test_value_of, mock_sstable_store, TEST_KEYS_COUNT, }; - use crate::hummock::iterator::{HummockIterator, UnorderedMergeIteratorInner}; + use crate::hummock::iterator::{HummockIterator, MergeIterator}; use crate::hummock::BackwardSstableIterator; #[tokio::test] @@ -54,7 +54,7 @@ mod test { BackwardSstableIterator::new(table2, sstable_store), ]; - let mut mi = UnorderedMergeIteratorInner::new(iters); + let mut mi = MergeIterator::new(iters); let mut i = 3 * TEST_KEYS_COUNT; mi.rewind().await.unwrap(); while mi.is_valid() { @@ -107,7 +107,7 @@ mod test { BackwardSstableIterator::new(table2, sstable_store), ]; - let mut mi = UnorderedMergeIteratorInner::new(iters); + let mut mi = MergeIterator::new(iters); // right edge case mi.seek(iterator_test_key_of(0).to_ref()).await.unwrap(); @@ -173,7 +173,7 @@ mod test { BackwardSstableIterator::new(table0, sstable_store), ]; - let mut mi = UnorderedMergeIteratorInner::new(iters); + let mut mi = MergeIterator::new(iters); mi.rewind().await.unwrap(); let mut count = 0; diff --git a/src/storage/src/hummock/iterator/backward_user.rs b/src/storage/src/hummock/iterator/backward_user.rs index 86f32407f20d5..d837f74efadf7 100644 --- a/src/storage/src/hummock/iterator/backward_user.rs +++ b/src/storage/src/hummock/iterator/backward_user.rs @@ -311,7 +311,7 @@ mod tests { iterator_test_bytes_user_key_of, iterator_test_user_key_of, iterator_test_value_of, mock_sstable_store, TEST_KEYS_COUNT, }; - use crate::hummock::iterator::UnorderedMergeIteratorInner; + use crate::hummock::iterator::MergeIterator; use crate::hummock::test_utils::gen_test_sstable; use crate::hummock::value::HummockValue; use crate::hummock::{BackwardSstableIterator, SstableStoreRef, TableHolder}; @@ -350,7 +350,7 @@ mod tests { BackwardSstableIterator::new(table0, sstable_store), ]; - let mi = UnorderedMergeIteratorInner::new(backward_iters); + let mi = MergeIterator::new(backward_iters); let mut ui = BackwardUserIterator::for_test(mi, (Unbounded, Unbounded)); let mut i = 3 * TEST_KEYS_COUNT; ui.rewind().await.unwrap(); @@ -401,7 +401,7 @@ mod tests { BackwardSstableIterator::new(table2, sstable_store), ]; - let bmi = UnorderedMergeIteratorInner::new(backward_iters); + let bmi = MergeIterator::new(backward_iters); let mut bui = BackwardUserIterator::for_test(bmi, (Unbounded, Unbounded)); // right edge case @@ -460,7 +460,7 @@ mod tests { BackwardSstableIterator::new(table0, sstable_store.clone()), BackwardSstableIterator::new(table1, sstable_store), ]; - let bmi = UnorderedMergeIteratorInner::new(backward_iters); + let bmi = MergeIterator::new(backward_iters); let mut bui = BackwardUserIterator::for_test(bmi, (Unbounded, Unbounded)); bui.rewind().await.unwrap(); @@ -503,7 +503,7 @@ mod tests { let sstable = gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await; let backward_iters = vec![BackwardSstableIterator::new(sstable, sstable_store)]; - let bmi = UnorderedMergeIteratorInner::new(backward_iters); + let bmi = MergeIterator::new(backward_iters); let begin_key = Included(iterator_test_bytes_user_key_of(2)); let end_key = Included(iterator_test_bytes_user_key_of(7)); @@ -580,7 +580,7 @@ mod tests { let sstable = gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await; let backward_iters = vec![BackwardSstableIterator::new(sstable, sstable_store)]; - let bmi = UnorderedMergeIteratorInner::new(backward_iters); + let bmi = MergeIterator::new(backward_iters); let begin_key = Excluded(iterator_test_bytes_user_key_of(2)); let end_key = Included(iterator_test_bytes_user_key_of(7)); @@ -658,7 +658,7 @@ mod tests { let sstable = gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await; let backward_iters = vec![BackwardSstableIterator::new(sstable, sstable_store)]; - let bmi = UnorderedMergeIteratorInner::new(backward_iters); + let bmi = MergeIterator::new(backward_iters); let end_key = Included(iterator_test_bytes_user_key_of(7)); let mut bui = BackwardUserIterator::for_test(bmi, (Unbounded, end_key)); @@ -734,7 +734,7 @@ mod tests { let handle = gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await; let backward_iters = vec![BackwardSstableIterator::new(handle, sstable_store)]; - let bmi = UnorderedMergeIteratorInner::new(backward_iters); + let bmi = MergeIterator::new(backward_iters); let begin_key = Included(iterator_test_bytes_user_key_of(2)); let mut bui = BackwardUserIterator::for_test(bmi, (begin_key, Unbounded)); @@ -830,7 +830,7 @@ mod tests { }; let backward_iters = vec![BackwardSstableIterator::new(handle, sstable_store)]; - let bmi = UnorderedMergeIteratorInner::new(backward_iters); + let bmi = MergeIterator::new(backward_iters); let mut bui = BackwardUserIterator::for_test(bmi, (start_bound, end_bound)); let num_puts: usize = truth .iter() @@ -1071,7 +1071,7 @@ mod tests { let backward_iters = vec![BackwardSstableIterator::new(table0, sstable_store)]; let min_epoch = test_epoch((TEST_KEYS_COUNT / 5) as u64); - let mi = UnorderedMergeIteratorInner::new(backward_iters); + let mi = MergeIterator::new(backward_iters); let mut ui = BackwardUserIterator::with_min_epoch(mi, (Unbounded, Unbounded), min_epoch); ui.rewind().await.unwrap(); diff --git a/src/storage/src/hummock/iterator/forward_merge.rs b/src/storage/src/hummock/iterator/forward_merge.rs index 82c43afd0c11c..8219a7eb52823 100644 --- a/src/storage/src/hummock/iterator/forward_merge.rs +++ b/src/storage/src/hummock/iterator/forward_merge.rs @@ -28,117 +28,84 @@ mod test { gen_merge_iterator_interleave_test_sstable_iters, iterator_test_key_of, iterator_test_value_of, mock_sstable_store, TEST_KEYS_COUNT, }; - use crate::hummock::iterator::{ - Forward, HummockIterator, HummockIteratorUnion, OrderedMergeIteratorInner, - UnorderedMergeIteratorInner, - }; + use crate::hummock::iterator::{Forward, HummockIterator, MergeIterator}; use crate::hummock::sstable::{ SstableIterator, SstableIteratorReadOptions, SstableIteratorType, }; - use crate::hummock::test_utils::gen_test_sstable; use crate::hummock::value::HummockValue; use crate::hummock::HummockResult; use crate::monitor::StoreLocalStatistic; #[tokio::test] async fn test_merge_basic() { - let mut unordered_iter: HummockIteratorUnion< - Forward, - UnorderedMergeIteratorInner, - OrderedMergeIteratorInner, - > = HummockIteratorUnion::First(UnorderedMergeIteratorInner::new( - gen_merge_iterator_interleave_test_sstable_iters(TEST_KEYS_COUNT, 3).await, - )); - let mut ordered_iter: HummockIteratorUnion< - Forward, - UnorderedMergeIteratorInner, - OrderedMergeIteratorInner, - > = HummockIteratorUnion::Second(OrderedMergeIteratorInner::new( + let mut iter = MergeIterator::new( gen_merge_iterator_interleave_test_sstable_iters(TEST_KEYS_COUNT, 3).await, - )); + ); - // Test both ordered and unordered iterators - let test_iters = vec![&mut unordered_iter, &mut ordered_iter]; - for iter in test_iters { - let mut i = 0; - iter.rewind().await.unwrap(); - while iter.is_valid() { - let key = iter.key(); - let val = iter.value(); - assert_eq!(key, iterator_test_key_of(i).to_ref()); - assert_eq!( - val.into_user_value().unwrap(), - iterator_test_value_of(i).as_slice() - ); - i += 1; - iter.next().await.unwrap(); - if i == TEST_KEYS_COUNT * 3 { - assert!(!iter.is_valid()); - break; - } + // Test merge iterators + let mut i = 0; + iter.rewind().await.unwrap(); + while iter.is_valid() { + let key = iter.key(); + let val = iter.value(); + assert_eq!(key, iterator_test_key_of(i).to_ref()); + assert_eq!( + val.into_user_value().unwrap(), + iterator_test_value_of(i).as_slice() + ); + i += 1; + iter.next().await.unwrap(); + if i == TEST_KEYS_COUNT * 3 { + assert!(!iter.is_valid()); + break; } - assert!(i >= TEST_KEYS_COUNT * 3); } + assert!(i >= TEST_KEYS_COUNT * 3); } #[tokio::test] async fn test_merge_seek() { - let mut unordered_iter: HummockIteratorUnion< - Forward, - UnorderedMergeIteratorInner, - OrderedMergeIteratorInner, - > = HummockIteratorUnion::First(UnorderedMergeIteratorInner::new( - gen_merge_iterator_interleave_test_sstable_iters(TEST_KEYS_COUNT, 3).await, - )); - let mut ordered_iter: HummockIteratorUnion< - Forward, - UnorderedMergeIteratorInner, - OrderedMergeIteratorInner, - > = HummockIteratorUnion::Second(OrderedMergeIteratorInner::new( + let mut iter = MergeIterator::new( gen_merge_iterator_interleave_test_sstable_iters(TEST_KEYS_COUNT, 3).await, - )); - - // Test both ordered and unordered iterators - let test_iters = vec![&mut unordered_iter, &mut ordered_iter]; - - for iter in test_iters { - // right edge case - iter.seek(iterator_test_key_of(TEST_KEYS_COUNT * 3).to_ref()) - .await - .unwrap(); - assert!(!iter.is_valid()); - - // normal case - iter.seek(iterator_test_key_of(TEST_KEYS_COUNT * 2 + 5).to_ref()) - .await - .unwrap(); - let k = iter.key(); - let v = iter.value(); - assert_eq!( - v.into_user_value().unwrap(), - iterator_test_value_of(TEST_KEYS_COUNT * 2 + 5).as_slice() - ); - assert_eq!(k, iterator_test_key_of(TEST_KEYS_COUNT * 2 + 5).to_ref()); - - iter.seek(iterator_test_key_of(17).to_ref()).await.unwrap(); - let k = iter.key(); - let v = iter.value(); - assert_eq!( - v.into_user_value().unwrap(), - iterator_test_value_of(TEST_KEYS_COUNT + 7).as_slice() - ); - assert_eq!(k, iterator_test_key_of(TEST_KEYS_COUNT + 7).to_ref()); - - // left edge case - iter.seek(iterator_test_key_of(0).to_ref()).await.unwrap(); - let k = iter.key(); - let v = iter.value(); - assert_eq!( - v.into_user_value().unwrap(), - iterator_test_value_of(0).as_slice() - ); - assert_eq!(k, iterator_test_key_of(0).to_ref()); - } + ); + + // Test merge iterators + // right edge case + iter.seek(iterator_test_key_of(TEST_KEYS_COUNT * 3).to_ref()) + .await + .unwrap(); + assert!(!iter.is_valid()); + + // normal case + iter.seek(iterator_test_key_of(TEST_KEYS_COUNT * 2 + 5).to_ref()) + .await + .unwrap(); + let k = iter.key(); + let v = iter.value(); + assert_eq!( + v.into_user_value().unwrap(), + iterator_test_value_of(TEST_KEYS_COUNT * 2 + 5).as_slice() + ); + assert_eq!(k, iterator_test_key_of(TEST_KEYS_COUNT * 2 + 5).to_ref()); + + iter.seek(iterator_test_key_of(17).to_ref()).await.unwrap(); + let k = iter.key(); + let v = iter.value(); + assert_eq!( + v.into_user_value().unwrap(), + iterator_test_value_of(TEST_KEYS_COUNT + 7).as_slice() + ); + assert_eq!(k, iterator_test_key_of(TEST_KEYS_COUNT + 7).to_ref()); + + // left edge case + iter.seek(iterator_test_key_of(0).to_ref()).await.unwrap(); + let k = iter.key(); + let v = iter.value(); + assert_eq!( + v.into_user_value().unwrap(), + iterator_test_value_of(0).as_slice() + ); + assert_eq!(k, iterator_test_key_of(0).to_ref()); } #[tokio::test] @@ -163,27 +130,7 @@ mod test { .await; let mut stats = StoreLocalStatistic::default(); - let mut unordered_iter: HummockIteratorUnion< - Forward, - UnorderedMergeIteratorInner, - OrderedMergeIteratorInner, - > = HummockIteratorUnion::First(UnorderedMergeIteratorInner::new(vec![ - SstableIterator::create( - sstable_store.sstable(&table0, &mut stats).await.unwrap(), - sstable_store.clone(), - read_options.clone(), - ), - SstableIterator::create( - sstable_store.sstable(&table1, &mut stats).await.unwrap(), - sstable_store.clone(), - read_options.clone(), - ), - ])); - let mut ordered_iter: HummockIteratorUnion< - Forward, - UnorderedMergeIteratorInner, - OrderedMergeIteratorInner, - > = HummockIteratorUnion::Second(OrderedMergeIteratorInner::new(vec![ + let mut iter = MergeIterator::new(vec![ SstableIterator::create( sstable_store.sstable(&table0, &mut stats).await.unwrap(), sstable_store.clone(), @@ -194,109 +141,23 @@ mod test { sstable_store.clone(), read_options.clone(), ), - ])); - - // Test both ordered and unordered iterators - let test_iters = vec![&mut unordered_iter, &mut ordered_iter]; - - for iter in test_iters { - iter.rewind().await.unwrap(); - let mut count = 0; - while iter.is_valid() { - count += 1; - iter.next().await.unwrap(); - } - assert_eq!(count, TEST_KEYS_COUNT * 2); - - iter.rewind().await.unwrap(); - let mut count = 0; - while iter.is_valid() { - count += 1; - iter.next().await.unwrap(); - } - assert_eq!(count, TEST_KEYS_COUNT * 2); - } - } - - #[tokio::test] - async fn test_ordered_merge_iter() { - let sstable_store = mock_sstable_store(); - let read_options = Arc::new(SstableIteratorReadOptions::default()); - - let non_overlapped_sstable = gen_test_sstable( - default_builder_opt_for_test(), - 0, - (0..TEST_KEYS_COUNT).filter(|x| x % 3 == 0).map(|x| { - ( - iterator_test_key_of(x), - HummockValue::put(format!("non_overlapped_{}", x).as_bytes().to_vec()), - ) - }), - sstable_store.clone(), - ) - .await; - - let overlapped_old_sstable = gen_test_sstable( - default_builder_opt_for_test(), - 1, - (0..TEST_KEYS_COUNT).filter(|x| x % 3 != 0).map(|x| { - ( - iterator_test_key_of(x), - HummockValue::put(format!("overlapped_old_{}", x).as_bytes().to_vec()), - ) - }), - sstable_store.clone(), - ) - .await; - - let overlapped_new_sstable = gen_test_sstable( - default_builder_opt_for_test(), - 2, - (0..TEST_KEYS_COUNT).filter(|x| x % 3 == 1).map(|x| { - ( - iterator_test_key_of(x), - HummockValue::put(format!("overlapped_new_{}", x).as_bytes().to_vec()), - ) - }), - sstable_store.clone(), - ) - .await; - let mut iter = OrderedMergeIteratorInner::new(vec![ - SstableIterator::create( - non_overlapped_sstable, - sstable_store.clone(), - read_options.clone(), - ), - SstableIterator::create( - overlapped_new_sstable, - sstable_store.clone(), - read_options.clone(), - ), - SstableIterator::create( - overlapped_old_sstable, - sstable_store.clone(), - read_options.clone(), - ), ]); iter.rewind().await.unwrap(); - let mut count = 0; - while iter.is_valid() { - assert_eq!(iter.key(), iterator_test_key_of(count).to_ref()); - let expected_value = match count % 3 { - 0 => format!("non_overlapped_{}", count).as_bytes().to_vec(), - 1 => format!("overlapped_new_{}", count).as_bytes().to_vec(), - 2 => format!("overlapped_old_{}", count).as_bytes().to_vec(), - _ => unreachable!(), - }; - assert_eq!(iter.value(), HummockValue::put(expected_value.as_slice())); count += 1; iter.next().await.unwrap(); } + assert_eq!(count, TEST_KEYS_COUNT * 2); - assert_eq!(count, TEST_KEYS_COUNT); + iter.rewind().await.unwrap(); + let mut count = 0; + while iter.is_valid() { + count += 1; + iter.next().await.unwrap(); + } + assert_eq!(count, TEST_KEYS_COUNT * 2); } struct CancellationTestIterator {} @@ -339,9 +200,9 @@ mod test { #[tokio::test] async fn test_merge_iter_cancel() { - let mut merge_iter = UnorderedMergeIteratorInner::new(vec![ - OrderedMergeIteratorInner::new(once(CancellationTestIterator {})), - OrderedMergeIteratorInner::new(once(CancellationTestIterator {})), + let mut merge_iter = MergeIterator::new(vec![ + MergeIterator::new(once(CancellationTestIterator {})), + MergeIterator::new(once(CancellationTestIterator {})), ]); merge_iter.rewind().await.unwrap(); let future = merge_iter.next(); @@ -354,7 +215,7 @@ mod test { .is_pending()); } - // Dropping the future will panic if the OrderedMergeIteratorInner is not cancellation safe. + // Dropping the future will panic if the OrderedMergeIterator is not cancellation safe. // See https://github.com/risingwavelabs/risingwave/issues/6637 } } diff --git a/src/storage/src/hummock/iterator/forward_user.rs b/src/storage/src/hummock/iterator/forward_user.rs index efefb45eccd85..97a9c2765d2a2 100644 --- a/src/storage/src/hummock/iterator/forward_user.rs +++ b/src/storage/src/hummock/iterator/forward_user.rs @@ -16,7 +16,7 @@ use std::ops::Bound::*; use bytes::Bytes; use risingwave_common::util::epoch::MAX_SPILL_TIMES; -use risingwave_hummock_sdk::key::{FullKey, UserKey, UserKeyRange}; +use risingwave_hummock_sdk::key::{FullKey, FullKeyTracker, UserKey, UserKeyRange}; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch}; use super::DeleteRangeIterator; @@ -31,11 +31,11 @@ pub struct UserIterator> { /// Inner table iterator. iterator: I, - /// Last full key. - last_key: FullKey, + // Track the last seen full key + full_key_tracker: FullKeyTracker, /// Last user value - last_val: Bytes, + latest_val: Bytes, /// Start and end bounds of user key. key_range: UserKeyRange, @@ -71,14 +71,14 @@ impl> UserIterator { Self { iterator, key_range, - last_key: FullKey::default(), - last_val: Bytes::new(), + latest_val: Bytes::new(), read_epoch, min_epoch, stats: StoreLocalStatistic::default(), delete_range_iter, _version: version, is_current_pos_valid: false, + full_key_tracker: FullKeyTracker::new(FullKey::default()), } } @@ -121,7 +121,7 @@ impl> UserIterator { /// Note: before call the function you need to ensure that the iterator is valid. pub fn key(&self) -> &FullKey { assert!(self.is_valid()); - &self.last_key + &self.full_key_tracker.latest_full_key } /// The returned value is in the form of user value. @@ -129,14 +129,14 @@ impl> UserIterator { /// Note: before call the function you need to ensure that the iterator is valid. pub fn value(&self) -> &Bytes { assert!(self.is_valid()); - &self.last_val + &self.latest_val } /// Resets the iterating position to the beginning. pub async fn rewind(&mut self) -> HummockResult<()> { // Reset self.is_current_pos_valid = false; - self.last_key = FullKey::default(); + self.full_key_tracker = FullKeyTracker::new(FullKey::default()); // Handle range scan match &self.key_range.0 { @@ -180,7 +180,7 @@ impl> UserIterator { pub async fn seek(&mut self, user_key: UserKey<&[u8]>) -> HummockResult<()> { // Reset self.is_current_pos_valid = false; - self.last_key = FullKey::default(); + self.full_key_tracker = FullKeyTracker::new(FullKey::default()); // Handle range scan when key < begin_key let user_key = match &self.key_range.0 { @@ -245,14 +245,13 @@ impl> UserIterator { } // Skip older version entry for the same user key - if self.last_key.user_key.as_ref() == full_key.user_key { + if self.full_key_tracker.observe(full_key).is_none() { self.stats.skip_multi_version_key_count += 1; self.iterator.next().await?; continue; } // A new user key is observed. - self.last_key = full_key.copy_into(); // It is better to early return here if the user key is already // out of range to avoid unnecessary access on the range tomestones @@ -271,7 +270,7 @@ impl> UserIterator { if self.delete_range_iter.current_epoch() >= epoch { self.stats.skip_delete_key_count += 1; } else { - self.last_val = Bytes::copy_from_slice(val); + self.latest_val = Bytes::copy_from_slice(val); self.stats.processed_key_count += 1; self.is_current_pos_valid = true; return Ok(()); @@ -336,7 +335,7 @@ mod tests { iterator_test_bytes_key_of_epoch, iterator_test_bytes_user_key_of, iterator_test_value_of, mock_sstable_store, TEST_KEYS_COUNT, }; - use crate::hummock::iterator::UnorderedMergeIteratorInner; + use crate::hummock::iterator::MergeIterator; use crate::hummock::sstable::{ SstableIterator, SstableIteratorReadOptions, SstableIteratorType, }; @@ -378,7 +377,7 @@ mod tests { SstableIterator::create(table2, sstable_store, read_options.clone()), ]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut ui = UserIterator::for_test(mi, (Unbounded, Unbounded)); ui.rewind().await.unwrap(); @@ -432,7 +431,7 @@ mod tests { SstableIterator::create(table2, sstable_store, read_options), ]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut ui = UserIterator::for_test(mi, (Unbounded, Unbounded)); // right edge case @@ -495,7 +494,7 @@ mod tests { SstableIterator::create(table1, sstable_store.clone(), read_options), ]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut ui = UserIterator::for_test(mi, (Unbounded, Unbounded)); ui.rewind().await.unwrap(); @@ -551,7 +550,7 @@ mod tests { let table = generate_test_data(sstable_store.clone(), vec![]).await; let read_options = Arc::new(SstableIteratorReadOptions::default()); let iters = vec![SstableIterator::create(table, sstable_store, read_options)]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let begin_key = Included(iterator_test_bytes_user_key_of(2)); let end_key = Included(iterator_test_bytes_user_key_of(7)); @@ -629,7 +628,7 @@ mod tests { gen_iterator_test_sstable_from_kv_pair(0, kv_pairs, sstable_store.clone()).await; let read_options = Arc::new(SstableIteratorReadOptions::default()); let iters = vec![SstableIterator::create(table, sstable_store, read_options)]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let begin_key = Included(iterator_test_bytes_user_key_of(2)); let end_key = Excluded(iterator_test_bytes_user_key_of(7)); @@ -692,7 +691,7 @@ mod tests { let table = generate_test_data(sstable_store.clone(), vec![]).await; let read_options = Arc::new(SstableIteratorReadOptions::default()); let iters = vec![SstableIterator::create(table, sstable_store, read_options)]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let end_key = Included(iterator_test_bytes_user_key_of(7)); let mut ui = UserIterator::for_test(mi, (Unbounded, end_key)); @@ -756,7 +755,7 @@ mod tests { let table = generate_test_data(sstable_store.clone(), vec![]).await; let read_options = Arc::new(SstableIteratorReadOptions::default()); let iters = vec![SstableIterator::create(table, sstable_store, read_options)]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let begin_key = Included(iterator_test_bytes_user_key_of(2)); let mut ui = UserIterator::for_test(mi, (begin_key, Unbounded)); @@ -836,7 +835,7 @@ mod tests { )]; let min_epoch = test_epoch((TEST_KEYS_COUNT / 5) as u64); - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut ui = UserIterator::for_test_with_epoch(mi, (Unbounded, Unbounded), u64::MAX, min_epoch); ui.rewind().await.unwrap(); @@ -870,7 +869,7 @@ mod tests { sstable_store.clone(), Arc::new(read_options), )]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut del_iter = ForwardMergeRangeIterator::new(150); del_iter.add_sst_iter(SstableDeleteRangeIterator::new(table.clone())); @@ -908,7 +907,7 @@ mod tests { )]; let mut del_iter = ForwardMergeRangeIterator::new(300); del_iter.add_sst_iter(SstableDeleteRangeIterator::new(table.clone())); - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut ui: UserIterator<_> = UserIterator::new( mi, (Unbounded, Unbounded), diff --git a/src/storage/src/hummock/iterator/merge_inner.rs b/src/storage/src/hummock/iterator/merge_inner.rs index c9639d9babad1..00c789a386bd7 100644 --- a/src/storage/src/hummock/iterator/merge_inner.rs +++ b/src/storage/src/hummock/iterator/merge_inner.rs @@ -14,35 +14,26 @@ use std::collections::binary_heap::PeekMut; use std::collections::{BinaryHeap, LinkedList}; -use std::future::Future; use std::ops::{Deref, DerefMut}; -use bytes::Bytes; -use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKey}; -use risingwave_hummock_sdk::EpochWithGap; +use futures::FutureExt; +use risingwave_hummock_sdk::key::FullKey; -use crate::hummock::iterator::{DirectionEnum, Forward, HummockIterator, HummockIteratorDirection}; -use crate::hummock::shared_buffer::shared_buffer_batch::SharedBufferBatchIterator; +use super::Forward; +use crate::hummock::iterator::{DirectionEnum, HummockIterator, HummockIteratorDirection}; +use crate::hummock::shared_buffer::shared_buffer_batch::{ + SharedBufferBatchIterator, SharedBufferVersionedEntryRef, +}; use crate::hummock::value::HummockValue; use crate::hummock::HummockResult; use crate::monitor::StoreLocalStatistic; -pub trait NodeExtraOrderInfo: Eq + Ord + Send + Sync {} - -/// For unordered merge iterator, no extra order info is needed. -type UnorderedNodeExtra = (); -/// Store the order index for the order aware merge iterator -type OrderedNodeExtra = usize; -impl NodeExtraOrderInfo for UnorderedNodeExtra {} -impl NodeExtraOrderInfo for OrderedNodeExtra {} - -pub struct Node { +pub struct Node { iter: I, - extra_order_info: T, } -impl Eq for Node where Self: PartialEq {} -impl PartialOrd for Node +impl Eq for Node where Self: PartialEq {} +impl PartialOrd for Node where Self: Ord, { @@ -52,7 +43,7 @@ where } /// Implement `Ord` for unordered iter node. Only compare the key. -impl Ord for Node { +impl Ord for Node { fn cmp(&self, other: &Self) -> std::cmp::Ordering { // Note: to implement min-heap by using max-heap internally, the comparing // order should be reversed. @@ -64,91 +55,22 @@ impl Ord for Node { } } -/// Implement `Ord` for ordered iter node. Compare key and use order index as tie breaker. -impl Ord for Node { - fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // The `extra_info` is used as a tie-breaker when the keys are equal. - match I::Direction::direction() { - DirectionEnum::Forward => other - .iter - .key() - .cmp(&self.iter.key()) - .then_with(|| other.extra_order_info.cmp(&self.extra_order_info)), - DirectionEnum::Backward => self - .iter - .key() - .cmp(&other.iter.key()) - .then_with(|| self.extra_order_info.cmp(&other.extra_order_info)), - } - } -} - -impl PartialEq for Node { +impl PartialEq for Node { fn eq(&self, other: &Self) -> bool { self.iter.key() == other.iter.key() } } -impl PartialEq for Node { - fn eq(&self, other: &Self) -> bool { - self.iter.key() == other.iter.key() && self.extra_order_info.eq(&other.extra_order_info) - } -} - /// Iterates on multiple iterators, a.k.a. `MergeIterator`. -pub struct MergeIteratorInner { +pub struct MergeIterator { /// Invalid or non-initialized iterators. - unused_iters: LinkedList>, + unused_iters: LinkedList>, /// The heap for merge sort. - heap: BinaryHeap>, - - last_table_key: Vec, + heap: BinaryHeap>, } -/// An order aware merge iterator. -#[allow(type_alias_bounds)] -pub type OrderedMergeIteratorInner = MergeIteratorInner; - -impl OrderedMergeIteratorInner { - pub fn new(iterators: impl IntoIterator) -> Self { - Self::create(iterators) - } - - pub fn for_compactor(iterators: impl IntoIterator) -> Self { - Self::create(iterators) - } - - fn create(iterators: impl IntoIterator) -> Self { - Self { - unused_iters: iterators - .into_iter() - .enumerate() - .map(|(i, iter)| Node { - iter, - extra_order_info: i, - }) - .collect(), - heap: BinaryHeap::new(), - last_table_key: Vec::new(), - } - } -} - -impl OrderedMergeIteratorInner> { - /// Used in `merge_imms_in_memory` to merge immutable memtables. - pub fn current_item(&self) -> (TableKey, (EpochWithGap, HummockValue)) { - let item = self - .heap - .peek() - .expect("no inner iter for imm merge") - .iter - .current_item(); - (item.0.clone(), (item.1 .0, item.1 .1.clone())) - } -} - -impl MergeIteratorInner { +impl MergeIterator { fn collect_local_statistic_impl(&self, stats: &mut StoreLocalStatistic) { for node in &self.heap { node.iter.collect_local_statistic(stats); @@ -159,11 +81,7 @@ impl MergeIteratorInner { } } -#[allow(type_alias_bounds)] -pub type UnorderedMergeIteratorInner = - MergeIteratorInner; - -impl UnorderedMergeIteratorInner { +impl MergeIterator { pub fn new(iterators: impl IntoIterator) -> Self { Self::create(iterators) } @@ -174,22 +92,26 @@ impl UnorderedMergeIteratorInner { fn create(iterators: impl IntoIterator) -> Self { Self { - unused_iters: iterators - .into_iter() - .map(|iter| Node { - iter, - extra_order_info: (), - }) - .collect(), + unused_iters: iterators.into_iter().map(|iter| Node { iter }).collect(), heap: BinaryHeap::new(), - last_table_key: Vec::new(), } } } -impl MergeIteratorInner +impl MergeIterator> { + /// Used in `merge_imms_in_memory` to merge immutable memtables. + pub(crate) fn current_key_entry(&self) -> SharedBufferVersionedEntryRef<'_> { + self.heap + .peek() + .expect("no inner iter for imm merge") + .iter + .current_key_entry() + } +} + +impl MergeIterator where - Node: Ord, + Node: Ord, { /// Moves all iterators from the `heap` to the linked list. fn reset_heap(&mut self) { @@ -208,12 +130,6 @@ where } } -/// The behaviour of `next` of order aware merge iterator is different from the normal one, so we -/// extract this trait. -trait MergeIteratorNext { - fn next_inner(&mut self) -> impl Future> + Send + '_; -} - /// This is a wrapper for the `PeekMut` of heap. /// /// Several panics due to future cancellation are caused by calling `drop` on the `PeekMut` when @@ -282,54 +198,38 @@ impl<'a, T: Ord> Drop for PeekMutGuard<'a, T> { } } -impl MergeIteratorNext for OrderedMergeIteratorInner { - async fn next_inner(&mut self) -> HummockResult<()> { - let top_key = { - let top_key = self.heap.peek().expect("no inner iter").iter.key(); - self.last_table_key.clear(); - self.last_table_key - .extend_from_slice(top_key.user_key.table_key.0); - FullKey { - user_key: UserKey { - table_id: top_key.user_key.table_id, - table_key: TableKey(self.last_table_key.as_slice()), - }, - epoch_with_gap: top_key.epoch_with_gap, - } - }; - loop { - let Some(mut node) = PeekMutGuard::peek_mut(&mut self.heap, &mut self.unused_iters) - else { - break; - }; - // WARNING: within scope of BinaryHeap::PeekMut, we must carefully handle all places - // of return. Once the iterator enters an invalid state, we should - // remove it from heap before returning. - - if node.iter.key() == top_key { - if let Err(e) = node.iter.next().await { - node.pop(); - self.heap.clear(); - return Err(e); - }; - if !node.iter.is_valid() { - let node = node.pop(); - self.unused_iters.push_back(node); - } else { - node.used(); - } - } else { - node.used(); - break; - } +impl MergeIterator> { + pub(crate) fn advance_peek_to_next_key(&mut self) { + let mut node = + PeekMutGuard::peek_mut(&mut self.heap, &mut self.unused_iters).expect("no inner iter"); + + node.iter.advance_to_next_key(); + + if !node.iter.is_valid() { + // Put back to `unused_iters` + let node = node.pop(); + self.unused_iters.push_back(node); + } else { + // This will update the heap top. + node.used(); } + } - Ok(()) + pub(crate) fn rewind_no_await(&mut self) { + self.rewind() + .now_or_never() + .expect("should not pending") + .expect("should not err") } } -impl MergeIteratorNext for UnorderedMergeIteratorInner { - async fn next_inner(&mut self) -> HummockResult<()> { +impl HummockIterator for MergeIterator +where + Node: Ord, +{ + type Direction = I::Direction; + + async fn next(&mut self) -> HummockResult<()> { let mut node = PeekMutGuard::peek_mut(&mut self.heap, &mut self.unused_iters).expect("no inner iter"); @@ -359,18 +259,6 @@ impl MergeIteratorNext for UnorderedMergeIteratorInner { Ok(()) } -} - -impl HummockIterator for MergeIteratorInner -where - Self: MergeIteratorNext, - Node: Ord, -{ - type Direction = I::Direction; - - async fn next(&mut self) -> HummockResult<()> { - self.next_inner().await - } fn key(&self) -> FullKey<&[u8]> { self.heap.peek().expect("no inner iter").iter.key() diff --git a/src/storage/src/hummock/iterator/mod.rs b/src/storage/src/hummock/iterator/mod.rs index ea7150a37ed9b..36318a10b4ae0 100644 --- a/src/storage/src/hummock/iterator/mod.rs +++ b/src/storage/src/hummock/iterator/mod.rs @@ -35,7 +35,7 @@ mod forward_merge; pub mod forward_user; mod merge_inner; pub use forward_user::*; -pub use merge_inner::{OrderedMergeIteratorInner, UnorderedMergeIteratorInner}; +pub use merge_inner::MergeIterator; use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKey}; use risingwave_hummock_sdk::EpochWithGap; diff --git a/src/storage/src/hummock/iterator/skip_watermark.rs b/src/storage/src/hummock/iterator/skip_watermark.rs index 5b230ac3732eb..e1c03f58beac7 100644 --- a/src/storage/src/hummock/iterator/skip_watermark.rs +++ b/src/storage/src/hummock/iterator/skip_watermark.rs @@ -30,21 +30,106 @@ use crate::monitor::StoreLocalStatistic; pub struct SkipWatermarkIterator { inner: I, - watermarks: BTreeMap, - remain_watermarks: VecDeque<(TableId, VirtualNode, WatermarkDirection, Bytes)>, + state: SkipWatermarkState, } impl> SkipWatermarkIterator { pub fn new(inner: I, watermarks: BTreeMap) -> Self { Self { inner, + state: SkipWatermarkState::new(watermarks), + } + } + + pub fn from_safe_epoch_watermarks( + inner: I, + safe_epoch_watermarks: &BTreeMap, + ) -> Self { + Self { + inner, + state: SkipWatermarkState::from_safe_epoch_watermarks(safe_epoch_watermarks), + } + } + + fn reset_watermark(&mut self) { + self.state.reset_watermark(); + } + + /// Advance the key until iterator invalid or the current key will not be filtered by the latest watermark. + /// Calling this method should ensure that the first remaining watermark has been advanced to the current key. + /// + /// Return a flag indicating whether should later advance the watermark. + async fn advance_key_and_watermark(&mut self) -> HummockResult<()> { + // advance key and watermark in an interleave manner until nothing + // changed after the method is called. + while self.inner.is_valid() { + if !self.state.should_delete(&self.inner.key()) { + break; + } + self.inner.next().await?; + } + Ok(()) + } +} + +impl> HummockIterator for SkipWatermarkIterator { + type Direction = Forward; + + async fn next(&mut self) -> HummockResult<()> { + self.inner.next().await?; + // Check whether there is any remaining watermark and return early to + // avoid calling the async `advance_key_and_watermark`, since in benchmark + // performance downgrade is observed without this early return. + if self.state.has_watermark() { + self.advance_key_and_watermark().await?; + } + Ok(()) + } + + fn key(&self) -> FullKey<&[u8]> { + self.inner.key() + } + + fn value(&self) -> HummockValue<&[u8]> { + self.inner.value() + } + + fn is_valid(&self) -> bool { + self.inner.is_valid() + } + + async fn rewind(&mut self) -> HummockResult<()> { + self.reset_watermark(); + self.inner.rewind().await?; + self.advance_key_and_watermark().await?; + Ok(()) + } + + async fn seek<'a>(&'a mut self, key: FullKey<&'a [u8]>) -> HummockResult<()> { + self.reset_watermark(); + self.inner.seek(key).await?; + self.advance_key_and_watermark().await?; + Ok(()) + } + + fn collect_local_statistic(&self, stats: &mut StoreLocalStatistic) { + self.inner.collect_local_statistic(stats) + } +} +pub struct SkipWatermarkState { + watermarks: BTreeMap, + remain_watermarks: VecDeque<(TableId, VirtualNode, WatermarkDirection, Bytes)>, +} + +impl SkipWatermarkState { + pub fn new(watermarks: BTreeMap) -> Self { + Self { remain_watermarks: VecDeque::new(), watermarks, } } pub fn from_safe_epoch_watermarks( - inner: I, safe_epoch_watermarks: &BTreeMap, ) -> Self { let watermarks = safe_epoch_watermarks @@ -85,10 +170,36 @@ impl> SkipWatermarkIterator { ) }) .collect(); - Self::new(inner, watermarks) + Self::new(watermarks) } - fn reset_watermark(&mut self) { + #[inline(always)] + pub fn has_watermark(&self) -> bool { + !self.remain_watermarks.is_empty() + } + + pub fn should_delete(&mut self, key: &FullKey<&[u8]>) -> bool { + if let Some((table_id, vnode, direction, watermark)) = self.remain_watermarks.front() { + let key_table_id = key.user_key.table_id; + let (key_vnode, inner_key) = key.user_key.table_key.split_vnode(); + match (&key_table_id, &key_vnode).cmp(&(table_id, vnode)) { + Ordering::Less => { + return false; + } + Ordering::Equal => { + return direction.filter_by_watermark(inner_key, watermark); + } + Ordering::Greater => { + // The current key has advanced over the watermark. + // We may advance the watermark before advancing the key. + return self.advance_watermark(key); + } + } + } + false + } + + pub fn reset_watermark(&mut self) { self.remain_watermarks = self .watermarks .iter() @@ -112,163 +223,65 @@ impl> SkipWatermarkIterator { /// filter out the current or future key. /// /// Return a flag indicating whether the current key will be filtered by the current watermark. - fn advance_watermark(&mut self) -> bool { - if self.inner.is_valid() { - let key = self.inner.key(); - let key_table_id = key.user_key.table_id; - let (key_vnode, inner_key) = key.user_key.table_key.split_vnode(); - while let Some((table_id, vnode, direction, watermark)) = self.remain_watermarks.front() - { - match (table_id, vnode).cmp(&(&key_table_id, &key_vnode)) { - Ordering::Less => { - self.remain_watermarks.pop_front(); - continue; - } - Ordering::Equal => { - match direction { - WatermarkDirection::Ascending => { - match inner_key.cmp(watermark.as_ref()) { - Ordering::Less => { - // The current key will be filtered by the watermark. - // Return true to further advance the key. - return true; - } - Ordering::Equal | Ordering::Greater => { - // The current key has passed the watermark. - // Advance the next watermark. - self.remain_watermarks.pop_front(); - // Since it is impossible for a (table_id, vnode) tuple to have multiple - // watermark, after the pop_front, the next (table_id, vnode) must have - // exceeded the current key, and we can directly return and mark that the - // current key is not filtered by the watermark at the front. - #[cfg(debug_assertions)] + fn advance_watermark(&mut self, key: &FullKey<&[u8]>) -> bool { + let key_table_id = key.user_key.table_id; + let (key_vnode, inner_key) = key.user_key.table_key.split_vnode(); + while let Some((table_id, vnode, direction, watermark)) = self.remain_watermarks.front() { + match (table_id, vnode).cmp(&(&key_table_id, &key_vnode)) { + Ordering::Less => { + self.remain_watermarks.pop_front(); + continue; + } + Ordering::Equal => { + match direction { + WatermarkDirection::Ascending => { + match inner_key.cmp(watermark.as_ref()) { + Ordering::Less => { + // The current key will be filtered by the watermark. + // Return true to further advance the key. + return true; + } + Ordering::Equal | Ordering::Greater => { + // The current key has passed the watermark. + // Advance the next watermark. + self.remain_watermarks.pop_front(); + // Since it is impossible for a (table_id, vnode) tuple to have multiple + // watermark, after the pop_front, the next (table_id, vnode) must have + // exceeded the current key, and we can directly return and mark that the + // current key is not filtered by the watermark at the front. + #[cfg(debug_assertions)] + { + if let Some((next_table_id, next_vnode, _, _)) = + self.remain_watermarks.front() { - if let Some((next_table_id, next_vnode, _, _)) = - self.remain_watermarks.front() - { - assert!( - (next_table_id, next_vnode) - > (&key_table_id, &key_vnode) - ); - } + assert!( + (next_table_id, next_vnode) + > (&key_table_id, &key_vnode) + ); } - return false; } + return false; } } - WatermarkDirection::Descending => { - return match inner_key.cmp(watermark.as_ref()) { - // Current key as not reached the watermark. Just return. - Ordering::Less | Ordering::Equal => false, - // Current key will be filtered by the watermark. - // Return true to further advance the key. - Ordering::Greater => true, - }; - } + } + WatermarkDirection::Descending => { + return match inner_key.cmp(watermark.as_ref()) { + // Current key as not reached the watermark. Just return. + Ordering::Less | Ordering::Equal => false, + // Current key will be filtered by the watermark. + // Return true to further advance the key. + Ordering::Greater => true, + }; } } - Ordering::Greater => { - return false; - } + } + Ordering::Greater => { + return false; } } } false } - - /// Advance the key until iterator invalid or the current key will not be filtered by the latest watermark. - /// Calling this method should ensure that the first remaining watermark has been advanced to the current key. - /// - /// Return a flag indicating whether should later advance the watermark. - async fn advance_key(&mut self) -> HummockResult { - if let Some((table_id, vnode, direction, watermark)) = self.remain_watermarks.front() { - while self.inner.is_valid() { - { - let key = self.inner.key(); - let key_table_id = key.user_key.table_id; - let (key_vnode, inner_key) = key.user_key.table_key.split_vnode(); - match (&key_table_id, &key_vnode).cmp(&(table_id, vnode)) { - Ordering::Less => { - return Ok(false); - } - Ordering::Equal => { - if direction.filter_by_watermark(inner_key, watermark) { - self.inner.next().await?; - } else { - return Ok(false); - } - } - Ordering::Greater => { - // The current key has advanced over the watermark. - // We may return to advance the watermark before advancing the key. - return Ok(true); - } - } - }; - } - } - Ok(false) - } - - async fn advance_key_and_watermark(&mut self) -> HummockResult<()> { - // advance key and watermark in an interleave manner until nothing - // changed after the method is called. - loop { - if !self.advance_key().await? { - break; - } - - if !self.advance_watermark() { - break; - } - } - Ok(()) - } -} - -impl> HummockIterator for SkipWatermarkIterator { - type Direction = Forward; - - async fn next(&mut self) -> HummockResult<()> { - self.inner.next().await?; - // Check whether there is any remaining watermark and return early to - // avoid calling the async `advance_key_and_watermark`, since in benchmark - // performance downgrade is observed without this early return. - if !self.remain_watermarks.is_empty() { - self.advance_key_and_watermark().await?; - } - Ok(()) - } - - fn key(&self) -> FullKey<&[u8]> { - self.inner.key() - } - - fn value(&self) -> HummockValue<&[u8]> { - self.inner.value() - } - - fn is_valid(&self) -> bool { - self.inner.is_valid() - } - - async fn rewind(&mut self) -> HummockResult<()> { - self.reset_watermark(); - self.inner.rewind().await?; - self.advance_key_and_watermark().await?; - Ok(()) - } - - async fn seek<'a>(&'a mut self, key: FullKey<&'a [u8]>) -> HummockResult<()> { - self.reset_watermark(); - self.inner.seek(key).await?; - self.advance_key_and_watermark().await?; - Ok(()) - } - - fn collect_local_statistic(&self, stats: &mut StoreLocalStatistic) { - self.inner.collect_local_statistic(stats) - } } #[cfg(test)] @@ -293,7 +306,7 @@ mod tests { const TABLE_ID: TableId = TableId::new(233); async fn assert_iter_eq( - mut first: impl HummockIterator, + mut first: Option, mut second: impl HummockIterator, seek_key: Option<(usize, usize)>, ) { @@ -306,29 +319,40 @@ mod tests { }, epoch_with_gap: EpochWithGap::new_from_epoch(EPOCH), }; - first.seek(full_key.to_ref()).await.unwrap(); + if let Some(first) = &mut first { + first.seek(full_key.to_ref()).await.unwrap(); + } second.seek(full_key.to_ref()).await.unwrap() } else { - first.rewind().await.unwrap(); + if let Some(first) = &mut first { + first.rewind().await.unwrap(); + } second.rewind().await.unwrap(); } - while first.is_valid() { - assert!(second.is_valid()); - let first_key = first.key(); - let second_key = second.key(); - assert_eq!(first_key, second_key); - assert_eq!(first.value(), second.value()); - first.next().await.unwrap(); - second.next().await.unwrap(); + if let Some(first) = &mut first { + while first.is_valid() { + assert!(second.is_valid()); + let first_key = first.key(); + let second_key = second.key(); + assert_eq!(first_key, second_key); + assert_eq!(first.value(), second.value()); + first.next().await.unwrap(); + second.next().await.unwrap(); + } } assert!(!second.is_valid()); } fn build_batch( pairs: impl Iterator, HummockValue)>, - ) -> SharedBufferBatch { - SharedBufferBatch::for_test(pairs.collect(), EPOCH, TABLE_ID) + ) -> Option { + let pairs: Vec<_> = pairs.collect(); + if pairs.is_empty() { + None + } else { + Some(SharedBufferBatch::for_test(pairs, EPOCH, TABLE_ID)) + } } fn filter_with_watermarks( @@ -378,10 +402,12 @@ mod tests { read_watermark.clone(), )); let iter = SkipWatermarkIterator::new( - build_batch(items.clone().into_iter()).into_forward_iter(), + build_batch(items.clone().into_iter()) + .unwrap() + .into_forward_iter(), BTreeMap::from_iter(once((TABLE_ID, read_watermark.clone()))), ); - (batch.into_forward_iter(), iter) + (batch.map(|batch| batch.into_forward_iter()), iter) }; let (first, second) = gen_iters(); assert_iter_eq(first, second, None).await; diff --git a/src/storage/src/hummock/iterator/test_utils.rs b/src/storage/src/hummock/iterator/test_utils.rs index 6593953e50d2e..7cd5bac54a479 100644 --- a/src/storage/src/hummock/iterator/test_utils.rs +++ b/src/storage/src/hummock/iterator/test_utils.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use bytes::Bytes; use itertools::Itertools; use risingwave_common::catalog::TableId; +use risingwave_common::config::{MetricLevel, ObjectStoreConfig}; use risingwave_common::util::epoch::test_epoch; use risingwave_hummock_sdk::key::{FullKey, TableKey, UserKey}; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId}; @@ -34,9 +35,9 @@ use crate::hummock::test_utils::{ }; use crate::hummock::{ DeleteRangeTombstone, FileCache, HummockValue, SstableBuilderOptions, SstableIterator, - SstableIteratorType, SstableStoreRef, TableHolder, + SstableIteratorType, SstableStoreConfig, SstableStoreRef, TableHolder, }; -use crate::monitor::ObjectStoreMetrics; +use crate::monitor::{global_hummock_state_store_metrics, ObjectStoreMetrics}; /// `assert_eq` two `Vec` with human-readable format. #[macro_export] @@ -54,24 +55,28 @@ pub const TEST_KEYS_COUNT: usize = 10; pub fn mock_sstable_store() -> SstableStoreRef { mock_sstable_store_with_object_store(Arc::new(ObjectStoreImpl::InMem( - InMemObjectStore::new().monitored(Arc::new(ObjectStoreMetrics::unused())), + InMemObjectStore::new().monitored( + Arc::new(ObjectStoreMetrics::unused()), + ObjectStoreConfig::default(), + ), ))) } pub fn mock_sstable_store_with_object_store(store: ObjectStoreRef) -> SstableStoreRef { let path = "test".to_string(); - Arc::new(SstableStore::new( + Arc::new(SstableStore::new(SstableStoreConfig { store, path, - 64 << 20, - 64 << 20, - 0, - 64 << 20, - 16, - FileCache::none(), - FileCache::none(), - None, - )) + block_cache_capacity: 64 << 20, + meta_cache_capacity: 64 << 20, + high_priority_ratio: 0, + prefetch_buffer_capacity: 64 << 20, + max_prefetch_block_number: 16, + data_file_cache: FileCache::none(), + meta_file_cache: FileCache::none(), + recent_filter: None, + state_store_metrics: Arc::new(global_hummock_state_store_metrics(MetricLevel::Disabled)), + })) } pub fn iterator_test_table_key_of(idx: usize) -> Vec { diff --git a/src/storage/src/hummock/local_version/pinned_version.rs b/src/storage/src/hummock/local_version/pinned_version.rs index 419bcfc6f5155..46ef8edc442b3 100644 --- a/src/storage/src/hummock/local_version/pinned_version.rs +++ b/src/storage/src/hummock/local_version/pinned_version.rs @@ -25,6 +25,7 @@ use risingwave_hummock_sdk::{CompactionGroupId, HummockVersionId, INVALID_VERSIO use risingwave_pb::hummock::hummock_version::Levels; use risingwave_pb::hummock::PbLevel; use risingwave_rpc_client::HummockMetaClient; +use thiserror_ext::AsReport; use tokio::sync::mpsc::error::TryRecvError; use tokio::sync::mpsc::{UnboundedReceiver, UnboundedSender}; use tokio_retry::strategy::jitter; @@ -274,8 +275,8 @@ pub(crate) async fn start_pinned_version_worker( Err(err) => { let retry_after = retry_backoff.next().unwrap_or(max_retry_interval); tracing::warn!( - "Failed to unpin version {:?}. Will retry after about {} milliseconds", - err, + error = %err.as_report(), + "Failed to unpin version. Will retry after about {} milliseconds", retry_after.as_millis() ); tokio::time::sleep(retry_after).await; diff --git a/src/storage/src/hummock/mod.rs b/src/storage/src/hummock/mod.rs index e57bd176be7de..9efb22675f072 100644 --- a/src/storage/src/hummock/mod.rs +++ b/src/storage/src/hummock/mod.rs @@ -80,7 +80,7 @@ pub async fn get_from_sstable_info( // contain `TablePrefix` and `VnodePrefix`. if let Some(hash) = dist_key_hash && !hit_sstable_bloom_filter( - sstable.value(), + &sstable, &( Bound::Included(full_key.user_key), Bound::Included(full_key.user_key), @@ -90,10 +90,7 @@ pub async fn get_from_sstable_info( ) { if !read_options.ignore_range_tombstone { - let delete_epoch = get_min_delete_range_epoch_from_sstable( - sstable.value().as_ref(), - full_key.user_key, - ); + let delete_epoch = get_min_delete_range_epoch_from_sstable(&sstable, full_key.user_key); if delete_epoch <= full_key.epoch_with_gap.pure_epoch() { return Ok(Some(( HummockValue::Delete, @@ -116,10 +113,8 @@ pub async fn get_from_sstable_info( // Iterator has sought passed the borders. if !iter.is_valid() { if !read_options.ignore_range_tombstone { - let delete_epoch = get_min_delete_range_epoch_from_sstable( - iter.sst().value().as_ref(), - full_key.user_key, - ); + let delete_epoch = + get_min_delete_range_epoch_from_sstable(iter.sst(), full_key.user_key); if delete_epoch <= full_key.epoch_with_gap.pure_epoch() { return Ok(Some(( HummockValue::Delete, @@ -136,8 +131,7 @@ pub async fn get_from_sstable_info( let value = if iter.key().user_key == full_key.user_key { Some((iter.value().to_bytes(), iter.key().epoch_with_gap)) } else if !read_options.ignore_range_tombstone { - let delete_epoch = - get_min_delete_range_epoch_from_sstable(iter.sst().value().as_ref(), full_key.user_key); + let delete_epoch = get_min_delete_range_epoch_from_sstable(iter.sst(), full_key.user_key); if delete_epoch <= full_key.epoch_with_gap.pure_epoch() { Some(( HummockValue::Delete, diff --git a/src/storage/src/hummock/observer_manager.rs b/src/storage/src/hummock/observer_manager.rs index 2921eb064de9f..4e10d9a523950 100644 --- a/src/storage/src/hummock/observer_manager.rs +++ b/src/storage/src/hummock/observer_manager.rs @@ -26,14 +26,14 @@ use tokio::sync::mpsc::UnboundedSender; use crate::filter_key_extractor::{FilterKeyExtractorImpl, FilterKeyExtractorManagerRef}; use crate::hummock::backup_reader::BackupReaderRef; -use crate::hummock::event_handler::{HummockEvent, HummockVersionUpdate}; +use crate::hummock::event_handler::HummockVersionUpdate; use crate::hummock::write_limiter::WriteLimiterRef; pub struct HummockObserverNode { filter_key_extractor_manager: FilterKeyExtractorManagerRef, backup_reader: BackupReaderRef, write_limiter: WriteLimiterRef, - version_update_sender: UnboundedSender, + version_update_sender: UnboundedSender, version: u64, } @@ -71,17 +71,15 @@ impl ObserverState for HummockObserverNode { Info::HummockVersionDeltas(hummock_version_deltas) => { let _ = self .version_update_sender - .send(HummockEvent::VersionUpdate( - HummockVersionUpdate::VersionDeltas( - hummock_version_deltas - .version_deltas - .iter() - .map(HummockVersionDelta::from_rpc_protobuf) - .collect(), - ), + .send(HummockVersionUpdate::VersionDeltas( + hummock_version_deltas + .version_deltas + .iter() + .map(HummockVersionDelta::from_rpc_protobuf) + .collect(), )) .inspect_err(|e| { - tracing::error!("unable to send version delta: {:?}", e); + tracing::error!(event = ?e.0, "unable to send version delta"); }); } @@ -123,15 +121,15 @@ impl ObserverState for HummockObserverNode { ); let _ = self .version_update_sender - .send(HummockEvent::VersionUpdate( - HummockVersionUpdate::PinnedVersion(HummockVersion::from_rpc_protobuf( + .send(HummockVersionUpdate::PinnedVersion( + HummockVersion::from_rpc_protobuf( &snapshot .hummock_version .expect("should get hummock version"), - )), + ), )) .inspect_err(|e| { - tracing::error!("unable to send full version: {:?}", e); + tracing::error!(event = ?e.0, "unable to send full version"); }); let snapshot_version = snapshot.version.unwrap(); self.version = snapshot_version.catalog_version; @@ -142,7 +140,7 @@ impl HummockObserverNode { pub fn new( filter_key_extractor_manager: FilterKeyExtractorManagerRef, backup_reader: BackupReaderRef, - version_update_sender: UnboundedSender, + version_update_sender: UnboundedSender, write_limiter: WriteLimiterRef, ) -> Self { Self { diff --git a/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs b/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs index 94980d32e21ec..0469326a62d1e 100644 --- a/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs +++ b/src/storage/src/hummock/shared_buffer/shared_buffer_batch.rs @@ -16,12 +16,13 @@ use std::cmp::Ordering; use std::fmt::Debug; use std::future::Future; use std::marker::PhantomData; -use std::ops::{Bound, Deref, RangeBounds}; +use std::ops::Bound::Included; +use std::ops::{Bound, RangeBounds}; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering::Relaxed; use std::sync::{Arc, LazyLock}; -use bytes::{Bytes, BytesMut}; +use bytes::Bytes; use itertools::Itertools; use risingwave_common::catalog::TableId; use risingwave_common::hash::VirtualNode; @@ -40,47 +41,55 @@ use crate::mem_table::ImmId; use crate::storage_value::StorageValue; use crate::store::ReadOptions; -fn whether_update_largest_key, Q: AsRef<[u8]>>( - current_largest_key: &Bound

, - key_to_update: &Q, -) -> bool { - match current_largest_key { - Bound::Excluded(x) => x.as_ref() <= key_to_update.as_ref(), - Bound::Included(x) => x.as_ref() < key_to_update.as_ref(), - Bound::Unbounded => false, - } -} - /// The key is `table_key`, which does not contain table id or epoch. pub(crate) type SharedBufferItem = (TableKey, HummockValue); pub type SharedBufferBatchId = u64; -/// A shared buffer may contain data from multiple epochs, -/// there are multiple versions for a given key (`table_key`), we put those versions into a vector -/// and sort them in descending order, aka newest to oldest. -pub type SharedBufferVersionedEntry = (TableKey, Vec<(EpochWithGap, HummockValue)>); -type PointRangePair = (PointRange>, PointRange>); - -struct SharedBufferDeleteRangeMeta { - // smallest/largest keys below are only inferred from tombstones. - smallest_empty: bool, - smallest_table_key: BytesMut, - largest_table_key: Bound, - point_range_pairs: Vec, +pub(crate) type VersionedSharedBufferValue = (EpochWithGap, HummockValue); + +pub(crate) struct SharedBufferVersionedEntryRef<'a> { + pub(crate) key: &'a TableKey, + pub(crate) new_values: &'a [VersionedSharedBufferValue], +} + +#[derive(PartialEq, Debug)] +pub(crate) struct SharedBufferKeyEntry { + pub(crate) key: TableKey, + /// A shared buffer may contain data from multiple epochs for a specific key. + /// The values of all keys are stored together in the field `new_values` of `SharedBufferBatchInner` + /// as a single vector. `value_offset` is the starting offset of values of the current `key` in the `new_values` vector. + /// The end offset is the `value_offset` of the next entry or the vector end if the current entry is not the last one. + pub(crate) value_offset: usize, +} + +impl SharedBufferKeyEntry { + /// Return an exclusive offset of the values of key of index `i` + fn value_end_offset<'a>( + i: usize, + entries: &'a [SharedBufferKeyEntry], + values: &'a [VersionedSharedBufferValue], + ) -> usize { + entries + .get(i + 1) + .map(|entry| entry.value_offset) + .unwrap_or(values.len()) + } + + fn values<'a>( + i: usize, + entries: &'a [SharedBufferKeyEntry], + values: &'a [VersionedSharedBufferValue], + ) -> &'a [VersionedSharedBufferValue] { + &values[entries[i].value_offset..Self::value_end_offset(i, entries, values)] + } } #[derive(Debug)] pub(crate) struct SharedBufferBatchInner { - payload: Vec, - /// The list of imm ids that are merged into this batch - /// This field is immutable - imm_ids: Vec, + entries: Vec, + new_values: Vec, /// The epochs of the data in batch, sorted in ascending order (old to new) epochs: Vec, - monotonic_tombstone_events: Vec, - largest_table_key: Bound, - smallest_table_key: Bytes, - kv_count: usize, /// Total size of all key-value items (excluding the `epoch` of value versions) size: usize, _tracker: Option, @@ -91,192 +100,72 @@ pub(crate) struct SharedBufferBatchInner { impl SharedBufferBatchInner { pub(crate) fn new( - table_id: TableId, epoch: HummockEpoch, spill_offset: u16, payload: Vec, - delete_ranges: Vec<(Bound, Bound)>, size: usize, _tracker: Option, ) -> Self { - let point_range_pairs = delete_ranges - .into_iter() - .map(|(left_bound, right_bound)| { - ( - match left_bound { - Bound::Excluded(x) => PointRange::from_user_key( - UserKey::new(table_id, TableKey(x.to_vec())), - true, - ), - Bound::Included(x) => PointRange::from_user_key( - UserKey::new(table_id, TableKey(x.to_vec())), - false, - ), - Bound::Unbounded => unreachable!(), - }, - match right_bound { - Bound::Excluded(x) => PointRange::from_user_key( - UserKey::new(table_id, TableKey(x.to_vec())), - false, - ), - Bound::Included(x) => PointRange::from_user_key( - UserKey::new(table_id, TableKey(x.to_vec())), - true, - ), - Bound::Unbounded => PointRange::from_user_key( - UserKey::new( - TableId::new(table_id.table_id() + 1), - TableKey::default(), - ), - false, - ), - }, - ) - }) - .collect_vec(); + assert!(!payload.is_empty()); + debug_assert!(payload.iter().is_sorted_by_key(|(key, _)| key)); - let SharedBufferDeleteRangeMeta { - smallest_empty, - mut smallest_table_key, - mut largest_table_key, - point_range_pairs, - } = Self::get_table_key_ends(table_id, point_range_pairs); - - if let Some(item) = payload.last() { - if whether_update_largest_key(&largest_table_key, &item.0) { - largest_table_key = Bound::Included(item.0.clone().0); - } - } - if let Some(item) = payload.first() { - if smallest_empty || item.0.as_ref().lt(smallest_table_key.as_ref()) { - smallest_table_key.clear(); - smallest_table_key.extend_from_slice(item.0.as_ref()); - } - } - let kv_count = payload.len(); let epoch_with_gap = EpochWithGap::new(epoch, spill_offset); - let items = payload - .into_iter() - .map(|(k, v)| (k, vec![(epoch_with_gap, v)])) - .collect_vec(); - - let mut monotonic_tombstone_events = Vec::with_capacity(point_range_pairs.len() * 2); - for (start_point_range, end_point_range) in point_range_pairs { - monotonic_tombstone_events.push(MonotonicDeleteEvent { - event_key: start_point_range, - new_epoch: epoch, - }); - monotonic_tombstone_events.push(MonotonicDeleteEvent { - event_key: end_point_range, - new_epoch: HummockEpoch::MAX, + let mut entries = Vec::with_capacity(payload.len()); + let mut new_values = Vec::with_capacity(payload.len()); + for (i, (key, value)) in payload.into_iter().enumerate() { + entries.push(SharedBufferKeyEntry { + key, + value_offset: i, }); + new_values.push((epoch_with_gap, value)); } let batch_id = SHARED_BUFFER_BATCH_ID_GENERATOR.fetch_add(1, Relaxed); SharedBufferBatchInner { - payload: items, - imm_ids: vec![batch_id], + entries, + new_values, epochs: vec![epoch], - monotonic_tombstone_events, - kv_count, size, - largest_table_key, - smallest_table_key: smallest_table_key.freeze(), _tracker, batch_id, } } + pub fn values(&self, i: usize) -> &[VersionedSharedBufferValue] { + SharedBufferKeyEntry::values(i, &self.entries, &self.new_values) + } + #[allow(clippy::too_many_arguments)] pub(crate) fn new_with_multi_epoch_batches( epochs: Vec, - payload: Vec, - smallest_table_key: Bytes, - largest_table_key: Bound, - num_items: usize, - imm_ids: Vec, - monotonic_tombstone_events: Vec, + entries: Vec, + new_values: Vec, size: usize, + imm_id: ImmId, tracker: Option, ) -> Self { - debug_assert!(!imm_ids.is_empty()); + assert!(new_values.len() >= entries.len()); + assert!(!entries.is_empty()); + debug_assert!(entries.iter().is_sorted_by_key(|entry| &entry.key)); + debug_assert!(entries.iter().is_sorted_by_key(|entry| &entry.value_offset)); + debug_assert!((0..entries.len()).all(|i| SharedBufferKeyEntry::values( + i, + &entries, + &new_values + ) + .iter() + .rev() + .is_sorted_by_key(|(epoch_with_gap, _)| epoch_with_gap))); debug_assert!(!epochs.is_empty()); debug_assert!(epochs.is_sorted()); - let max_imm_id = *imm_ids.iter().max().unwrap(); - Self { - payload, + entries, + new_values, epochs, - imm_ids, - monotonic_tombstone_events, - largest_table_key, - smallest_table_key, - kv_count: num_items, size, _tracker: tracker, - batch_id: max_imm_id, - } - } - - fn get_table_key_ends( - table_id: TableId, - mut range_tombstone_list: Vec, - ) -> SharedBufferDeleteRangeMeta { - let mut largest_table_key = Bound::Included(Bytes::new()); - let mut smallest_table_key = BytesMut::new(); - let mut smallest_empty = true; - if !range_tombstone_list.is_empty() { - range_tombstone_list.sort_by(|a, b| a.0.cmp(&b.0).then_with(|| b.1.cmp(&a.1))); - let mut range_tombstones: Vec = vec![]; - for (start_point_range, end_point_range) in range_tombstone_list { - if start_point_range.ge(&end_point_range) { - continue; - } - let end_point_range_table_id = end_point_range.left_user_key.table_id; - if end_point_range_table_id != table_id { - // It means that the right side of the tombstone is +inf. - assert_eq!(end_point_range_table_id.table_id(), table_id.table_id() + 1); - largest_table_key = Bound::Unbounded; - } else if whether_update_largest_key( - &largest_table_key, - &end_point_range.left_user_key.table_key.0, - ) { - largest_table_key = if end_point_range.is_exclude_left_key { - Bound::Included(Bytes::from( - end_point_range.left_user_key.table_key.0.clone(), - )) - } else { - Bound::Excluded(Bytes::from( - end_point_range.left_user_key.table_key.0.clone(), - )) - }; - } - if smallest_empty - || smallest_table_key.gt(&start_point_range.left_user_key.table_key.0) - { - smallest_table_key.clear(); - smallest_table_key - .extend_from_slice(&start_point_range.left_user_key.table_key.0); - smallest_empty = false; - } - if let Some(last) = range_tombstones.last_mut() { - if last.1.ge(&start_point_range) { - if last.1.lt(&end_point_range) { - last.1 = end_point_range; - } - continue; - } - } - range_tombstones.push((start_point_range, end_point_range)); - } - range_tombstone_list = range_tombstones; - } - SharedBufferDeleteRangeMeta { - smallest_empty, - smallest_table_key, - largest_table_key, - point_range_pairs: range_tombstone_list, + batch_id: imm_id, } } @@ -284,17 +173,18 @@ impl SharedBufferBatchInner { /// Return `HummockValue::Delete` if the key has been deleted by some epoch <= `read_epoch` fn get_value( &self, - table_id: TableId, table_key: TableKey<&[u8]>, read_epoch: HummockEpoch, - read_options: &ReadOptions, ) -> Option<(HummockValue, EpochWithGap)> { // Perform binary search on table key to find the corresponding entry - if let Ok(i) = self.payload.binary_search_by(|m| (m.0[..]).cmp(*table_key)) { - let item = &self.payload[i]; - assert_eq!(item.0.as_ref(), *table_key); + if let Ok(i) = self + .entries + .binary_search_by(|m| (m.key.as_ref()).cmp(*table_key)) + { + let entry = &self.entries[i]; + assert_eq!(entry.key.as_ref(), *table_key); // Scan to find the first version <= epoch - for (e, v) in &item.1 { + for (e, v) in self.values(i) { // skip invisible versions if read_epoch < e.pure_epoch() { continue; @@ -304,47 +194,13 @@ impl SharedBufferBatchInner { // cannot find a visible version } - if !read_options.ignore_range_tombstone { - let delete_epoch = self.get_min_delete_range_epoch(UserKey::new(table_id, table_key)); - if delete_epoch <= read_epoch { - Some(( - HummockValue::Delete, - EpochWithGap::new_from_epoch(delete_epoch), - )) - } else { - None - } - } else { - None - } - } - - fn get_min_delete_range_epoch(&self, query_user_key: UserKey<&[u8]>) -> HummockEpoch { - let query_extended_user_key = PointRange::from_user_key(query_user_key, false); - let idx = self.monotonic_tombstone_events.partition_point( - |MonotonicDeleteEvent { event_key, .. }| { - event_key.as_ref().le(&query_extended_user_key) - }, - ); - if idx == 0 { - HummockEpoch::MAX - } else { - self.monotonic_tombstone_events[idx - 1].new_epoch - } - } -} - -impl Deref for SharedBufferBatchInner { - type Target = [SharedBufferVersionedEntry]; - - fn deref(&self) -> &Self::Target { - self.payload.as_slice() + None } } impl PartialEq for SharedBufferBatchInner { fn eq(&self, other: &Self) -> bool { - self.payload == other.payload + self.entries == other.entries && self.new_values == other.new_values } } @@ -369,16 +225,14 @@ impl SharedBufferBatch { Self { inner: Arc::new(SharedBufferBatchInner::new( - table_id, epoch, 0, sorted_items, - vec![], size, None, )), table_id, - instance_id: LocalInstanceId::default(), + instance_id: SHARED_BUFFER_BATCH_ID_GENERATOR.fetch_add(1, Relaxed), } } @@ -432,7 +286,7 @@ impl SharedBufferBatch { && range_overlap( &(left, right), &self.start_table_key(), - self.end_table_key().as_ref(), + Included(&self.end_table_key()), ) } @@ -440,10 +294,6 @@ impl SharedBufferBatch { self.table_id } - pub fn is_merged_imm(&self) -> bool { - self.inner.epochs.len() > 1 - } - pub fn min_epoch(&self) -> HummockEpoch { *self.inner.epochs.first().unwrap() } @@ -452,33 +302,28 @@ impl SharedBufferBatch { *self.inner.epochs.last().unwrap() } - pub fn get_imm_ids(&self) -> &Vec { - debug_assert!(!self.inner.imm_ids.is_empty()); - &self.inner.imm_ids + pub fn key_count(&self) -> usize { + self.inner.entries.len() } - pub fn kv_count(&self) -> usize { - self.inner.kv_count + pub fn value_count(&self) -> usize { + self.inner.new_values.len() } pub fn get( &self, table_key: TableKey<&[u8]>, read_epoch: HummockEpoch, - read_options: &ReadOptions, + _read_options: &ReadOptions, ) -> Option<(HummockValue, EpochWithGap)> { - self.inner - .get_value(self.table_id, table_key, read_epoch, read_options) - } - - pub fn get_min_delete_range_epoch(&self, user_key: UserKey<&[u8]>) -> HummockEpoch { - self.inner.get_min_delete_range_epoch(user_key) + self.inner.get_value(table_key, read_epoch) } pub fn range_exists(&self, table_key_range: &TableKeyRange) -> bool { self.inner + .entries .binary_search_by(|m| { - let key = &m.0; + let key = &m.key; let too_left = match &table_key_range.0 { std::ops::Bound::Included(range_start) => range_start.as_ref() > key.as_ref(), std::ops::Bound::Excluded(range_start) => range_start.as_ref() >= key.as_ref(), @@ -514,35 +359,19 @@ impl SharedBufferBatch { self.into_directed_iter() } - pub fn delete_range_iter(&self) -> SharedBufferDeleteRangeIterator { - SharedBufferDeleteRangeIterator::new(self.inner.clone()) - } - - pub fn get_payload(&self) -> &[SharedBufferVersionedEntry] { - &self.inner - } - #[inline(always)] pub fn start_table_key(&self) -> TableKey<&[u8]> { - TableKey(&self.inner.smallest_table_key) + TableKey(self.inner.entries.first().expect("non-empty").key.as_ref()) } #[inline(always)] - pub fn raw_smallest_key(&self) -> &Bytes { - &self.inner.smallest_table_key + pub fn end_table_key(&self) -> TableKey<&[u8]> { + TableKey(self.inner.entries.last().expect("non-empty").key.as_ref()) } #[inline(always)] - pub fn end_table_key(&self) -> Bound> { - self.inner - .largest_table_key - .as_ref() - .map(|largest_key| TableKey(largest_key.as_ref())) - } - - #[inline(always)] - pub fn raw_largest_key(&self) -> &Bound { - &self.inner.largest_table_key + pub fn raw_largest_key(&self) -> &TableKey { + &self.inner.entries.last().expect("non-empty").key } /// return inclusive left endpoint, which means that all data in this batch should be larger or @@ -551,11 +380,6 @@ impl SharedBufferBatch { UserKey::new(self.table_id, self.start_table_key()) } - #[inline(always)] - pub fn has_range_tombstone(&self) -> bool { - !self.inner.monotonic_tombstone_events.is_empty() - } - pub fn size(&self) -> usize { self.inner.size } @@ -582,31 +406,18 @@ impl SharedBufferBatch { spill_offset: u16, sorted_items: Vec, size: usize, - delete_ranges: Vec<(Bound, Bound)>, table_id: TableId, - instance_id: Option, + instance_id: LocalInstanceId, tracker: Option, ) -> Self { - let inner = SharedBufferBatchInner::new( - table_id, - epoch, - spill_offset, - sorted_items, - delete_ranges, - size, - tracker, - ); + let inner = SharedBufferBatchInner::new(epoch, spill_offset, sorted_items, size, tracker); SharedBufferBatch { inner: Arc::new(inner), table_id, - instance_id: instance_id.unwrap_or_default(), + instance_id, } } - pub fn get_delete_range_tombstones(&self) -> Vec { - self.inner.monotonic_tombstone_events.clone() - } - pub fn collect_vnodes(&self) -> Vec { let mut vnodes = Vec::with_capacity(VirtualNode::COUNT); let mut next_vnode_id = 0; @@ -618,39 +429,53 @@ impl SharedBufferBatch { ); let idx = match self .inner - .payload - .binary_search_by(|m| (m.0[..]).cmp(seek_key.as_slice())) + .entries + .binary_search_by(|m| (m.key.as_ref()).cmp(seek_key.as_slice())) { Ok(idx) => idx, Err(idx) => idx, }; - if idx >= self.inner.payload.len() { + if idx >= self.inner.entries.len() { break; } - let item = &self.inner.payload[idx]; - if item.0.len() <= VirtualNode::SIZE { + let item = &self.inner.entries[idx]; + if item.key.len() <= VirtualNode::SIZE { break; } - let current_vnode_id = VirtualNode::from_be_bytes( - item.0.as_ref()[..VirtualNode::SIZE] - .try_into() - .expect("slice with incorrect length"), - ) - .to_index(); + let current_vnode_id = item.key.vnode_part().to_index(); vnodes.push(current_vnode_id); next_vnode_id = current_vnode_id + 1; } vnodes } + + #[cfg(any(test, feature = "test"))] + pub fn build_shared_buffer_batch_for_test( + epoch: HummockEpoch, + spill_offset: u16, + sorted_items: Vec, + size: usize, + table_id: TableId, + ) -> Self { + let inner = SharedBufferBatchInner::new(epoch, spill_offset, sorted_items, size, None); + SharedBufferBatch { + inner: Arc::new(inner), + table_id, + instance_id: LocalInstanceId::default(), + } + } } /// Iterate all the items in the shared buffer batch /// If there are multiple versions of a key, the iterator will return all versions pub struct SharedBufferBatchIterator { inner: Arc, - current_version_idx: i32, - // The index of the current entry in the payload - current_idx: usize, + /// The index of the current entry in the payload + current_entry_idx: usize, + /// The index of current value + current_value_idx: usize, + /// The exclusive end offset of the value index of current key. + value_end_offset: usize, table_id: TableId, _phantom: PhantomData, } @@ -659,43 +484,90 @@ impl SharedBufferBatchIterator { pub(crate) fn new(inner: Arc, table_id: TableId) -> Self { Self { inner, - current_idx: 0, - current_version_idx: 0, + current_entry_idx: 0, + current_value_idx: 0, + value_end_offset: 0, table_id, _phantom: Default::default(), } } - /// Return all values of the current key - pub(crate) fn current_versions(&self) -> &Vec<(EpochWithGap, HummockValue)> { - debug_assert!(self.current_idx < self.inner.len()); - let idx = match D::direction() { - DirectionEnum::Forward => self.current_idx, - DirectionEnum::Backward => self.inner.len() - self.current_idx - 1, - }; - &self.inner.get(idx).unwrap().1 + fn is_valid_entry_idx(&self) -> bool { + self.current_entry_idx < self.inner.entries.len() } - fn current_versions_len(&self) -> i32 { - if self.current_idx < self.inner.len() { - self.current_versions().len() as i32 + fn advance_to_next_entry(&mut self) { + debug_assert!(self.is_valid_entry_idx()); + match D::direction() { + DirectionEnum::Forward => { + self.current_entry_idx += 1; + } + DirectionEnum::Backward => { + if self.current_entry_idx == 0 { + self.current_entry_idx = self.inner.entries.len(); + } else { + self.current_entry_idx -= 1; + } + } + } + } + + fn reset_value_idx(&mut self) { + debug_assert!(self.is_valid_entry_idx()); + self.current_value_idx = self.inner.entries[self.current_entry_idx].value_offset; + self.value_end_offset = self.get_value_end_offset(); + } + + fn get_value_end_offset(&self) -> usize { + debug_assert!(self.is_valid_entry_idx()); + SharedBufferKeyEntry::value_end_offset( + self.current_entry_idx, + &self.inner.entries, + &self.inner.new_values, + ) + } + + fn assert_valid_idx(&self) { + debug_assert!(self.is_valid_entry_idx()); + debug_assert!( + self.current_value_idx >= self.inner.entries[self.current_entry_idx].value_offset + ); + debug_assert_eq!(self.value_end_offset, self.get_value_end_offset()); + debug_assert!(self.current_value_idx < self.value_end_offset); + } + + fn advance_to_next_value(&mut self) { + self.assert_valid_idx(); + + if self.current_value_idx + 1 < self.value_end_offset { + self.current_value_idx += 1; } else { - 0 + self.advance_to_next_entry(); + if self.is_valid_entry_idx() { + self.reset_value_idx(); + } } } +} - pub(crate) fn current_item(&self) -> (&TableKey, &(EpochWithGap, HummockValue)) { - assert!(self.is_valid(), "iterator is not valid"); - let (idx, version_idx) = match D::direction() { - DirectionEnum::Forward => (self.current_idx, self.current_version_idx), - DirectionEnum::Backward => ( - self.inner.len() - self.current_idx - 1, - self.current_version_idx, - ), - }; - let cur_entry = self.inner.get(idx).unwrap(); - let value = &cur_entry.1[version_idx as usize]; - (&cur_entry.0, value) +impl SharedBufferBatchIterator { + pub(crate) fn advance_to_next_key(&mut self) { + self.advance_to_next_entry(); + if self.is_valid_entry_idx() { + self.reset_value_idx(); + } + } + + pub(crate) fn current_key_entry(&self) -> SharedBufferVersionedEntryRef<'_> { + self.assert_valid_idx(); + debug_assert_eq!( + self.current_value_idx, + self.inner.entries[self.current_entry_idx].value_offset + ); + SharedBufferVersionedEntryRef { + key: &self.inner.entries[self.current_entry_idx].key, + new_values: &self.inner.new_values[self.current_value_idx..self.value_end_offset], + } } } @@ -703,58 +575,36 @@ impl HummockIterator for SharedBufferBatchIterator< type Direction = D; async fn next(&mut self) -> HummockResult<()> { - assert!(self.is_valid()); - match D::direction() { - DirectionEnum::Forward => { - // If the current key has more versions, we need to advance the value index - if self.current_version_idx + 1 < self.current_versions_len() { - self.current_version_idx += 1; - } else { - self.current_idx += 1; - self.current_version_idx = 0; - } - } - DirectionEnum::Backward => { - if self.current_version_idx > 0 { - self.current_version_idx -= 1; - } else { - self.current_idx += 1; - self.current_version_idx = self.current_versions_len() - 1; - } - } - } + self.advance_to_next_value(); Ok(()) } fn key(&self) -> FullKey<&[u8]> { - let (key, (epoch_with_gap, _)) = self.current_item(); - FullKey::new_with_gap_epoch(self.table_id, TableKey(key), *epoch_with_gap) + self.assert_valid_idx(); + let key = self.inner.entries[self.current_entry_idx].key.as_ref(); + let epoch_with_gap = self.inner.new_values[self.current_value_idx].0; + FullKey::new_with_gap_epoch(self.table_id, TableKey(key), epoch_with_gap) } fn value(&self) -> HummockValue<&[u8]> { - let (_, (_, value)) = self.current_item(); - value.as_slice() + self.assert_valid_idx(); + self.inner.new_values[self.current_value_idx].1.as_slice() } fn is_valid(&self) -> bool { - if self.current_idx >= self.inner.len() { - return false; - } - self.current_version_idx >= 0 - && self.current_version_idx < self.current_versions().len() as i32 + self.is_valid_entry_idx() } async fn rewind(&mut self) -> HummockResult<()> { - self.current_idx = 0; - match D::direction() { DirectionEnum::Forward => { - self.current_version_idx = 0; + self.current_entry_idx = 0; } DirectionEnum::Backward => { - self.current_version_idx = self.current_versions_len() - 1; + self.current_entry_idx = self.inner.entries.len() - 1; } - } + }; + self.reset_value_idx(); Ok(()) } @@ -764,66 +614,44 @@ impl HummockIterator for SharedBufferBatchIterator< // by table key. let partition_point = self .inner - .binary_search_by(|probe| probe.0[..].cmp(*key.user_key.table_key)); + .entries + .binary_search_by(|probe| probe.key.as_ref().cmp(*key.user_key.table_key)); let seek_key_epoch = key.epoch_with_gap; - match D::direction() { - DirectionEnum::Forward => match partition_point { - Ok(i) => { - self.current_idx = i; - // seek to the first version that is <= the seek key epoch - let mut idx: i32 = 0; - for (epoch_with_gap, _) in self.current_versions() { - if epoch_with_gap <= &seek_key_epoch { - break; - } - idx += 1; - } - - // Move onto the next key for forward iteration if seek key epoch is smaller - // than all versions - if idx >= self.current_versions().len() as i32 { - self.current_idx += 1; - self.current_version_idx = 0; - } else { - self.current_version_idx = idx; + match partition_point { + Ok(i) => { + self.current_entry_idx = i; + self.reset_value_idx(); + while self.current_value_idx < self.value_end_offset { + let epoch_with_gap = self.inner.new_values[self.current_value_idx].0; + if epoch_with_gap <= seek_key_epoch { + break; } + self.current_value_idx += 1; } - Err(i) => { - self.current_idx = i; - self.current_version_idx = 0; + if self.current_value_idx == self.value_end_offset { + self.advance_to_next_entry(); + if self.is_valid_entry_idx() { + self.reset_value_idx(); + } } - }, - DirectionEnum::Backward => { - match partition_point { - Ok(i) => { - self.current_idx = self.inner.len() - i - 1; - // seek from back to the first version that is >= seek_key_epoch - let values = self.current_versions(); - let mut idx: i32 = (values.len() - 1) as i32; - for (epoch_with_gap, _) in values.iter().rev() { - if epoch_with_gap >= &seek_key_epoch { - break; - } - idx -= 1; - } - - if idx < 0 { - self.current_idx += 1; - self.current_version_idx = self.current_versions_len() - 1; - } else { - self.current_version_idx = idx; - } + } + Err(i) => match D::direction() { + DirectionEnum::Forward => { + self.current_entry_idx = i; + if self.is_valid_entry_idx() { + self.reset_value_idx(); } - // Seek to one item before the seek partition_point: - // If i == 0, the iterator will be invalidated with self.current_idx == - // self.inner.len(). - Err(i) => { - self.current_idx = self.inner.len() - i; - self.current_version_idx = self.current_versions_len() - 1; + } + DirectionEnum::Backward => { + if i == 0 { + self.current_entry_idx = self.inner.entries.len(); + } else { + self.current_entry_idx = i - 1; + self.reset_value_idx(); } } - } - } + }, + }; Ok(()) } @@ -831,13 +659,67 @@ impl HummockIterator for SharedBufferBatchIterator< } pub struct SharedBufferDeleteRangeIterator { - inner: Arc, + monotonic_tombstone_events: Vec, next_idx: usize, } impl SharedBufferDeleteRangeIterator { - pub(crate) fn new(inner: Arc) -> Self { - Self { inner, next_idx: 0 } + #[cfg(any(test, feature = "test"))] + pub(crate) fn new( + epoch: HummockEpoch, + table_id: TableId, + delete_ranges: Vec<(Bound, Bound)>, + ) -> Self { + let point_range_pairs = delete_ranges + .into_iter() + .map(|(left_bound, right_bound)| { + ( + match left_bound { + Bound::Excluded(x) => PointRange::from_user_key( + UserKey::new(table_id, TableKey(x.to_vec())), + true, + ), + Bound::Included(x) => PointRange::from_user_key( + UserKey::new(table_id, TableKey(x.to_vec())), + false, + ), + Bound::Unbounded => unreachable!(), + }, + match right_bound { + Bound::Excluded(x) => PointRange::from_user_key( + UserKey::new(table_id, TableKey(x.to_vec())), + false, + ), + Bound::Included(x) => PointRange::from_user_key( + UserKey::new(table_id, TableKey(x.to_vec())), + true, + ), + Bound::Unbounded => PointRange::from_user_key( + UserKey::new( + TableId::new(table_id.table_id() + 1), + TableKey::default(), + ), + false, + ), + }, + ) + }) + .collect_vec(); + let mut monotonic_tombstone_events = Vec::with_capacity(point_range_pairs.len() * 2); + for (start_point_range, end_point_range) in point_range_pairs { + monotonic_tombstone_events.push(MonotonicDeleteEvent { + event_key: start_point_range, + new_epoch: epoch, + }); + monotonic_tombstone_events.push(MonotonicDeleteEvent { + event_key: end_point_range, + new_epoch: HummockEpoch::MAX, + }); + } + Self { + monotonic_tombstone_events, + next_idx: 0, + } } } @@ -847,14 +729,14 @@ impl DeleteRangeIterator for SharedBufferDeleteRangeIterator { type SeekFuture<'a> = impl Future> + 'a; fn next_extended_user_key(&self) -> PointRange<&[u8]> { - self.inner.monotonic_tombstone_events[self.next_idx] + self.monotonic_tombstone_events[self.next_idx] .event_key .as_ref() } fn current_epoch(&self) -> HummockEpoch { if self.next_idx > 0 { - self.inner.monotonic_tombstone_events[self.next_idx - 1].new_epoch + self.monotonic_tombstone_events[self.next_idx - 1].new_epoch } else { HummockEpoch::MAX } @@ -877,7 +759,7 @@ impl DeleteRangeIterator for SharedBufferDeleteRangeIterator { fn seek<'a>(&'a mut self, target_user_key: UserKey<&'a [u8]>) -> Self::SeekFuture<'a> { async move { let target_extended_user_key = PointRange::from_user_key(target_user_key, false); - self.next_idx = self.inner.monotonic_tombstone_events.partition_point( + self.next_idx = self.monotonic_tombstone_events.partition_point( |MonotonicDeleteEvent { event_key, .. }| { event_key.as_ref().le(&target_extended_user_key) }, @@ -887,7 +769,7 @@ impl DeleteRangeIterator for SharedBufferDeleteRangeIterator { } fn is_valid(&self) -> bool { - self.next_idx < self.inner.monotonic_tombstone_events.len() + self.next_idx < self.monotonic_tombstone_events.len() } } @@ -895,9 +777,8 @@ impl DeleteRangeIterator for SharedBufferDeleteRangeIterator { mod tests { use std::ops::Bound::{Excluded, Included}; - use risingwave_common::must_match; use risingwave_common::util::epoch::{ - test_epoch, EPOCH_AVAILABLE_BITS, EPOCH_INC_MIN_STEP_FOR_TEST, + test_epoch, EpochExt, EPOCH_AVAILABLE_BITS, EPOCH_INC_MIN_STEP_FOR_TEST, }; use risingwave_hummock_sdk::key::map_table_key_range; @@ -936,7 +817,7 @@ mod tests { shared_buffer_items[0].0 ); assert_eq!( - must_match!(shared_buffer_batch.end_table_key(), Bound::Included(table_key) => *table_key), + *shared_buffer_batch.end_table_key(), shared_buffer_items[2].0 ); @@ -993,31 +874,6 @@ mod tests { } output.reverse(); assert_eq!(output, shared_buffer_items); - - let batch = SharedBufferBatch::build_shared_buffer_batch( - epoch, - 0, - vec![], - 1, - vec![ - ( - Bound::Included(Bytes::from("a")), - Bound::Excluded(Bytes::from("c")), - ), - ( - Bound::Included(Bytes::from("b")), - Bound::Excluded(Bytes::from("d")), - ), - ], - TableId::new(0), - None, - None, - ); - assert_eq!(batch.start_table_key().as_ref(), "a".as_bytes()); - assert_eq!( - must_match!(batch.end_table_key(), Bound::Excluded(table_key) => *table_key), - "d".as_bytes() - ); } #[tokio::test] @@ -1134,9 +990,9 @@ mod tests { } assert!(!iter.is_valid()); - // BACKWARD: Seek to 2nd key with future epoch, expect first item to return + // BACKWARD: Seek to 2nd key with old epoch, expect first item to return let mut iter = shared_buffer_batch.clone().into_backward_iter(); - iter.seek(iterator_test_key_of_epoch(2, test_epoch(2)).to_ref()) + iter.seek(iterator_test_key_of_epoch(2, epoch.prev_epoch()).to_ref()) .await .unwrap(); assert!(iter.is_valid()); @@ -1146,9 +1002,9 @@ mod tests { iter.next().await.unwrap(); assert!(!iter.is_valid()); - // BACKWARD: Seek to 2nd key with old epoch, expect first two item to return + // BACKWARD: Seek to 2nd key with future epoch, expect first two item to return let mut iter = shared_buffer_batch.clone().into_backward_iter(); - iter.seek(iterator_test_key_of_epoch(2, epoch - EPOCH_INC_MIN_STEP_FOR_TEST).to_ref()) + iter.seek(iterator_test_key_of_epoch(2, epoch.next_epoch()).to_ref()) .await .unwrap(); for item in shared_buffer_items[0..=1].iter().rev() { @@ -1160,55 +1016,6 @@ mod tests { assert!(!iter.is_valid()); } - #[tokio::test] - async fn test_shared_buffer_batch_delete_range() { - let epoch = test_epoch(1); - let delete_ranges = vec![ - ( - Bound::Included(Bytes::from(b"aaa".to_vec())), - Bound::Excluded(Bytes::from(b"bbb".to_vec())), - ), - ( - Bound::Included(Bytes::from(b"ccc".to_vec())), - Bound::Excluded(Bytes::from(b"ddd".to_vec())), - ), - ( - Bound::Included(Bytes::from(b"ddd".to_vec())), - Bound::Excluded(Bytes::from(b"eee".to_vec())), - ), - ]; - let shared_buffer_batch = SharedBufferBatch::build_shared_buffer_batch( - epoch, - 0, - vec![], - 0, - delete_ranges, - Default::default(), - None, - None, - ); - assert_eq!( - epoch, - shared_buffer_batch - .get_min_delete_range_epoch(UserKey::new(Default::default(), TableKey(b"aaa"),)) - ); - assert_eq!( - HummockEpoch::MAX, - shared_buffer_batch - .get_min_delete_range_epoch(UserKey::new(Default::default(), TableKey(b"bbb"),)) - ); - assert_eq!( - epoch, - shared_buffer_batch - .get_min_delete_range_epoch(UserKey::new(Default::default(), TableKey(b"ddd"),)) - ); - assert_eq!( - HummockEpoch::MAX, - shared_buffer_batch - .get_min_delete_range_epoch(UserKey::new(Default::default(), TableKey(b"eee"),)) - ); - } - #[tokio::test] #[should_panic] async fn test_invalid_table_id() { @@ -1330,16 +1137,14 @@ mod tests { table_id, ); - let batch_items = vec![ + let batch_items = [ shared_buffer_items1, shared_buffer_items2, shared_buffer_items3, ]; // newer data comes first let imms = vec![imm3, imm2, imm1]; - let merged_imm = merge_imms_in_memory(table_id, 0, imms.clone(), None) - .await - .unwrap(); + let merged_imm = merge_imms_in_memory(table_id, 0, imms.clone(), None).await; // Point lookup for (i, items) in batch_items.iter().enumerate() { @@ -1431,172 +1236,14 @@ mod tests { )); backward_iter.next().await.unwrap(); } - output.reverse(); + let mut expected = vec![]; + for key_idx in (0..=2).rev() { + for epoch in (1..=3).rev() { + let item = batch_items[epoch - 1][key_idx].clone(); + expected.push(item); + } + } assert_eq!(expected, output); } } - - fn test_table_key_of(idx: usize) -> Vec { - format!("{:03}", idx).as_bytes().to_vec() - } - - #[tokio::test] - async fn test_merge_imms_delete_range() { - let table_id = TableId { table_id: 1004 }; - let epoch = test_epoch(1); - let delete_ranges = vec![ - ( - Bound::Included(Bytes::from(b"111".to_vec())), - Bound::Excluded(Bytes::from(b"222".to_vec())), - ), - ( - Bound::Included(Bytes::from(b"555".to_vec())), - Bound::Excluded(Bytes::from(b"777".to_vec())), - ), - ( - Bound::Included(Bytes::from(b"aaa".to_vec())), - Bound::Excluded(Bytes::from(b"ddd".to_vec())), - ), - ]; - let shared_buffer_items1: Vec<(Vec, HummockValue)> = vec![ - ( - test_table_key_of(222), - HummockValue::put(Bytes::from("value2")), - ), - ( - test_table_key_of(333), - HummockValue::put(Bytes::from("value3")), - ), - ( - test_table_key_of(888), - HummockValue::put(Bytes::from("value8")), - ), - ]; - let sorted_items1 = transform_shared_buffer(shared_buffer_items1); - let size = SharedBufferBatch::measure_batch_size(&sorted_items1); - let imm1 = SharedBufferBatch::build_shared_buffer_batch( - epoch, - 0, - sorted_items1, - size, - delete_ranges, - table_id, - None, - None, - ); - - let epoch = test_epoch(2); - let delete_ranges = vec![ - ( - Bound::Included(Bytes::from(b"444".to_vec())), - Bound::Excluded(Bytes::from(b"555".to_vec())), - ), - ( - Bound::Included(Bytes::from(b"888".to_vec())), - Bound::Excluded(Bytes::from(b"999".to_vec())), - ), - ( - Bound::Included(Bytes::from(b"bbb".to_vec())), - Bound::Excluded(Bytes::from(b"ccc".to_vec())), - ), - ]; - let shared_buffer_items2: Vec<(Vec, HummockValue)> = vec![ - ( - test_table_key_of(111), - HummockValue::put(Bytes::from("value12")), - ), - ( - test_table_key_of(222), - HummockValue::put(Bytes::from("value22")), - ), - ( - test_table_key_of(333), - HummockValue::put(Bytes::from("value32")), - ), - ( - test_table_key_of(555), - HummockValue::put(Bytes::from("value52")), - ), - ]; - let sorted_items2 = transform_shared_buffer(shared_buffer_items2); - let size = SharedBufferBatch::measure_batch_size(&sorted_items2); - let imm2 = SharedBufferBatch::build_shared_buffer_batch( - epoch, - 0, - sorted_items2, - size, - delete_ranges, - table_id, - None, - None, - ); - - let imms = vec![imm2, imm1]; - let merged_imm = merge_imms_in_memory(table_id, 0, imms, None).await.unwrap(); - - assert_eq!( - test_epoch(1), - merged_imm.get_min_delete_range_epoch(UserKey::new(table_id, TableKey(b"111"))) - ); - assert_eq!( - test_epoch(1), - merged_imm.get_min_delete_range_epoch(UserKey::new(table_id, TableKey(b"555"))) - ); - assert_eq!( - test_epoch(2), - merged_imm.get_min_delete_range_epoch(UserKey::new(table_id, TableKey(b"888"))) - ); - - assert_eq!( - HummockValue::put(Bytes::from("value12")), - merged_imm - .get(TableKey(b"111"), test_epoch(2), &ReadOptions::default()) - .unwrap() - .0 - ); - - // 555 is deleted in epoch=1 - assert_eq!( - HummockValue::Delete, - merged_imm - .get(TableKey(b"555"), test_epoch(1), &ReadOptions::default()) - .unwrap() - .0 - ); - - // 555 is inserted again in epoch=2 - assert_eq!( - HummockValue::put(Bytes::from("value52")), - merged_imm - .get(TableKey(b"555"), test_epoch(2), &ReadOptions::default()) - .unwrap() - .0 - ); - - // "666" is deleted in epoch=1 and isn't inserted in later epochs - assert_eq!( - HummockValue::Delete, - merged_imm - .get(TableKey(b"666"), test_epoch(2), &ReadOptions::default()) - .unwrap() - .0 - ); - // "888" is deleted in epoch=2 - assert_eq!( - HummockValue::Delete, - merged_imm - .get(TableKey(b"888"), test_epoch(2), &ReadOptions::default()) - .unwrap() - .0 - ); - - // 888 exists in the snapshot of epoch=1 - assert_eq!( - HummockValue::put(Bytes::from("value8")), - merged_imm - .get(TableKey(b"888"), test_epoch(1), &ReadOptions::default()) - .unwrap() - .0 - ); - } } diff --git a/src/storage/src/hummock/sstable/backward_sstable_iterator.rs b/src/storage/src/hummock/sstable/backward_sstable_iterator.rs index bf72b1a9dfbd9..62eff53e72655 100644 --- a/src/storage/src/hummock/sstable/backward_sstable_iterator.rs +++ b/src/storage/src/hummock/sstable/backward_sstable_iterator.rs @@ -35,7 +35,7 @@ pub struct BackwardSstableIterator { cur_idx: usize, /// Reference to the sstable - pub sst: TableHolder, + sst: TableHolder, sstable_store: SstableStoreRef, @@ -46,7 +46,7 @@ impl BackwardSstableIterator { pub fn new(sstable: TableHolder, sstable_store: SstableStoreRef) -> Self { Self { block_iter: None, - cur_idx: sstable.value().meta.block_metas.len() - 1, + cur_idx: sstable.meta.block_metas.len() - 1, sst: sstable, sstable_store, stats: StoreLocalStatistic::default(), @@ -59,13 +59,13 @@ impl BackwardSstableIterator { idx: isize, seek_key: Option>, ) -> HummockResult<()> { - if idx >= self.sst.value().block_count() as isize || idx < 0 { + if idx >= self.sst.block_count() as isize || idx < 0 { self.block_iter = None; } else { let block = self .sstable_store .get( - self.sst.value(), + &self.sst, idx as usize, crate::hummock::CachePolicy::Fill(CachePriority::High), &mut self.stats, @@ -117,14 +117,13 @@ impl HummockIterator for BackwardSstableIterator { /// Instead of setting idx to 0th block, a `BackwardSstableIterator` rewinds to the last block /// in the sstable. async fn rewind(&mut self) -> HummockResult<()> { - self.seek_idx(self.sst.value().block_count() as isize - 1, None) + self.seek_idx(self.sst.block_count() as isize - 1, None) .await } async fn seek<'a>(&'a mut self, key: FullKey<&'a [u8]>) -> HummockResult<()> { let block_idx = self .sst - .value() .meta .block_metas .partition_point(|block_meta| { @@ -187,7 +186,7 @@ mod tests { .await; // We should have at least 10 blocks, so that sstable iterator test could cover more code // path. - assert!(handle.value().meta.block_metas.len() > 10); + assert!(handle.meta.block_metas.len() > 10); let mut sstable_iter = BackwardSstableIterator::new(handle, sstable_store); let mut cnt = TEST_KEYS_COUNT; sstable_iter.rewind().await.unwrap(); @@ -212,7 +211,7 @@ mod tests { .await; // We should have at least 10 blocks, so that sstable iterator test could cover more code // path. - assert!(sstable.value().meta.block_metas.len() > 10); + assert!(sstable.meta.block_metas.len() > 10); let mut sstable_iter = BackwardSstableIterator::new(sstable, sstable_store); let mut all_key_to_test = (0..TEST_KEYS_COUNT).collect_vec(); let mut rng = thread_rng(); diff --git a/src/storage/src/hummock/sstable/block.rs b/src/storage/src/hummock/sstable/block.rs index 2748ce0556643..1028348a7d349 100644 --- a/src/storage/src/hummock/sstable/block.rs +++ b/src/storage/src/hummock/sstable/block.rs @@ -28,6 +28,7 @@ use super::utils::{bytes_diff_below_max_key_length, xxhash64_verify, Compression use crate::hummock::sstable::utils; use crate::hummock::sstable::utils::xxhash64_checksum; use crate::hummock::{HummockError, HummockResult}; +use crate::monitor::Hitmap; pub const DEFAULT_BLOCK_SIZE: usize = 4 * 1024; pub const DEFAULT_RESTART_INTERVAL: usize = 16; @@ -139,10 +140,9 @@ impl RestartPoint { } } -#[derive(Clone)] pub struct Block { /// Uncompressed entries data, with restart encoded restart points info. - pub data: Bytes, + data: Bytes, /// Uncompressed entried data length. data_len: usize, @@ -151,6 +151,20 @@ pub struct Block { /// Restart points. restart_points: Vec, + + hitmap: Hitmap<{ Self::HITMAP_ELEMS }>, +} + +impl Clone for Block { + fn clone(&self) -> Self { + Self { + data: self.data.clone(), + data_len: self.data_len, + table_id: self.table_id, + restart_points: self.restart_points.clone(), + hitmap: self.hitmap.clone(), + } + } } impl Debug for Block { @@ -164,6 +178,8 @@ impl Debug for Block { } impl Block { + pub const HITMAP_ELEMS: usize = 4; + pub fn get_algorithm(buf: &Bytes) -> HummockResult { let compression = CompressionAlgorithm::decode(&mut &buf[buf.len() - 9..buf.len() - 8])?; Ok(compression) @@ -190,29 +206,29 @@ impl Block { let buf = match compression { CompressionAlgorithm::None => { if copy { - buf.slice(0..(buf.len() - 9)) - } else { Bytes::copy_from_slice(&buf[0..(buf.len() - 9)]) + } else { + buf.slice(0..(buf.len() - 9)) } } CompressionAlgorithm::Lz4 => { let mut decoder = lz4::Decoder::new(compressed_data.reader()) .map_err(HummockError::decode_error)?; let mut decoded = Vec::with_capacity(uncompressed_capacity); - decoder + let read_size = decoder .read_to_end(&mut decoded) .map_err(HummockError::decode_error)?; - debug_assert_eq!(decoded.capacity(), uncompressed_capacity); + assert_eq!(read_size, uncompressed_capacity); Bytes::from(decoded) } CompressionAlgorithm::Zstd => { let mut decoder = zstd::Decoder::new(compressed_data.reader()) .map_err(HummockError::decode_error)?; let mut decoded = Vec::with_capacity(uncompressed_capacity); - decoder + let read_size = decoder .read_to_end(&mut decoded) .map_err(HummockError::decode_error)?; - debug_assert_eq!(decoded.capacity(), uncompressed_capacity); + assert_eq!(read_size, uncompressed_capacity); Bytes::from(decoded) } }; @@ -272,6 +288,7 @@ impl Block { data_len, restart_points, table_id: TableId::new(table_id), + hitmap: Hitmap::default(), } } @@ -314,9 +331,17 @@ impl Block { &self.data[..self.data_len] } - pub fn raw_data(&self) -> &[u8] { + pub fn raw(&self) -> &[u8] { &self.data[..] } + + pub fn hitmap(&self) -> &Hitmap<{ Self::HITMAP_ELEMS }> { + &self.hitmap + } + + pub fn efficiency(&self) -> f64 { + self.hitmap.ratio() + } } /// [`KeyPrefix`] contains info for prefix compression. @@ -420,6 +445,8 @@ impl Default for BlockBuilderOptions { pub struct BlockBuilder { /// Write buffer. buf: BytesMut, + /// Compress buffer + compress_buf: BytesMut, /// Entry interval between restart points. restart_count: usize, /// Restart points. @@ -440,8 +467,9 @@ pub struct BlockBuilder { impl BlockBuilder { pub fn new(options: BlockBuilderOptions) -> Self { Self { - // add more space to avoid re-allocate space. - buf: BytesMut::with_capacity(options.capacity + 256), + // add more space to avoid re-allocate space. (for restart_points and restart_points_type_index) + buf: BytesMut::with_capacity(Self::buf_reserve_size(&options)), + compress_buf: BytesMut::default(), restart_count: options.restart_interval, restart_points: Vec::with_capacity( options.capacity / DEFAULT_ENTRY_SIZE / options.restart_interval + 1, @@ -639,22 +667,35 @@ impl BlockBuilder { ); self.buf.put_u32_le(self.table_id.unwrap()); - if self.compression_algorithm != CompressionAlgorithm::None { - self.buf = Self::compress(&self.buf[..], self.compression_algorithm); - } + let result_buf = if self.compression_algorithm != CompressionAlgorithm::None { + self.compress_buf.clear(); + self.compress_buf = Self::compress( + &self.buf[..], + self.compression_algorithm, + std::mem::take(&mut self.compress_buf), + ); + + &mut self.compress_buf + } else { + &mut self.buf + }; - self.compression_algorithm.encode(&mut self.buf); - let checksum = xxhash64_checksum(&self.buf); - self.buf.put_u64_le(checksum); + self.compression_algorithm.encode(result_buf); + let checksum = xxhash64_checksum(result_buf); + result_buf.put_u64_le(checksum); assert!( - self.buf.len() < (u32::MAX) as usize, + result_buf.len() < (u32::MAX) as usize, "buf_len {} entry_count {} table {:?}", - self.buf.len(), + result_buf.len(), self.entry_count, self.table_id ); - self.buf.as_ref() + if self.compression_algorithm != CompressionAlgorithm::None { + self.compress_buf.as_ref() + } else { + self.buf.as_ref() + } } pub fn compress_block( @@ -668,21 +709,29 @@ impl BlockBuilder { let compression = CompressionAlgorithm::decode(&mut &buf[buf.len() - 9..buf.len() - 8])?; let compressed_data = &buf[..buf.len() - 9]; assert_eq!(compression, CompressionAlgorithm::None); - let mut writer = Self::compress(compressed_data, target_compression); + let mut compress_writer = Self::compress( + compressed_data, + target_compression, + BytesMut::with_capacity(buf.len()), + ); - target_compression.encode(&mut writer); - let checksum = xxhash64_checksum(&writer); - writer.put_u64_le(checksum); - Ok(writer.freeze()) + target_compression.encode(&mut compress_writer); + let checksum = xxhash64_checksum(&compress_writer); + compress_writer.put_u64_le(checksum); + Ok(compress_writer.freeze()) } - pub fn compress(buf: &[u8], compression_algorithm: CompressionAlgorithm) -> BytesMut { + pub fn compress( + buf: &[u8], + compression_algorithm: CompressionAlgorithm, + compress_writer: BytesMut, + ) -> BytesMut { match compression_algorithm { CompressionAlgorithm::None => unreachable!(), CompressionAlgorithm::Lz4 => { let mut encoder = lz4::EncoderBuilder::new() .level(4) - .build(BytesMut::with_capacity(buf.len()).writer()) + .build(compress_writer.writer()) .map_err(HummockError::encode_error) .unwrap(); encoder @@ -694,10 +743,9 @@ impl BlockBuilder { writer.into_inner() } CompressionAlgorithm::Zstd => { - let mut encoder = - zstd::Encoder::new(BytesMut::with_capacity(buf.len()).writer(), 4) - .map_err(HummockError::encode_error) - .unwrap(); + let mut encoder = zstd::Encoder::new(compress_writer.writer(), 4) + .map_err(HummockError::encode_error) + .unwrap(); encoder .write_all(buf) .map_err(HummockError::encode_error) @@ -737,6 +785,10 @@ impl BlockBuilder { pub fn table_id(&self) -> Option { self.table_id } + + fn buf_reserve_size(option: &BlockBuilderOptions) -> usize { + option.capacity + 1024 + 256 + } } #[cfg(test)] diff --git a/src/storage/src/hummock/sstable/block_iterator.rs b/src/storage/src/hummock/sstable/block_iterator.rs index 7580428114977..7344d033ddde4 100644 --- a/src/storage/src/hummock/sstable/block_iterator.rs +++ b/src/storage/src/hummock/sstable/block_iterator.rs @@ -19,8 +19,9 @@ use bytes::BytesMut; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::key::FullKey; -use super::{KeyPrefix, LenType, RestartPoint}; +use super::{Block, KeyPrefix, LenType, RestartPoint}; use crate::hummock::BlockHolder; +use crate::monitor::LocalHitmap; /// [`BlockIterator`] is used to read kv pairs in a block. pub struct BlockIterator { @@ -39,10 +40,23 @@ pub struct BlockIterator { last_key_len_type: LenType, last_value_len_type: LenType, + + /// NOTE: + /// + /// - `hitmap` is supposed to be updated each time accessing the block data in a new position. + /// - `hitmap` must be reported to the block hitmap before drop. + hitmap: LocalHitmap<{ Block::HITMAP_ELEMS }>, +} + +impl Drop for BlockIterator { + fn drop(&mut self) { + self.block.hitmap().report(&mut self.hitmap); + } } impl BlockIterator { pub fn new(block: BlockHolder) -> Self { + let hitmap = LocalHitmap::default(); Self { block, offset: usize::MAX, @@ -52,6 +66,7 @@ impl BlockIterator { entry_len: 0, last_key_len_type: LenType::u8, last_value_len_type: LenType::u8, + hitmap, } } @@ -81,7 +96,6 @@ impl BlockIterator { pub fn key(&self) -> FullKey<&[u8]> { assert!(self.is_valid()); - FullKey::from_slice_without_table_id(self.table_id(), &self.key[..]) } @@ -105,13 +119,11 @@ impl BlockIterator { pub fn seek(&mut self, key: FullKey<&[u8]>) { self.seek_restart_point_by_key(key); - self.next_until_key(key); } pub fn seek_le(&mut self, key: FullKey<&[u8]>) { self.seek_restart_point_by_key(key); - self.next_until_key(key); if !self.is_valid() { self.seek_to_last(); @@ -172,6 +184,9 @@ impl BlockIterator { self.offset = offset; self.entry_len = prefix.entry_len(); + self.hitmap + .fill_with_range(self.offset, self.value_range.end, self.block.len()); + true } @@ -241,27 +256,41 @@ impl BlockIterator { } /// Searches the restart point index that the given `key` belongs to. - fn search_restart_point_index_by_key(&self, key: FullKey<&[u8]>) -> usize { + fn search_restart_point_index_by_key(&mut self, key: FullKey<&[u8]>) -> usize { // Find the largest restart point that restart key equals or less than the given key. - self.block + let res = self + .block .search_restart_partition_point( |&RestartPoint { offset: probe, key_len_type, value_len_type, }| { - let prefix = - self.decode_prefix_at(probe as usize, key_len_type, value_len_type); + let probe = probe as usize; + let prefix = KeyPrefix::decode( + &mut &self.block.data()[probe..], + probe, + key_len_type, + value_len_type, + ); let probe_key = &self.block.data()[prefix.diff_key_range()]; let full_probe_key = FullKey::from_slice_without_table_id(self.block.table_id(), probe_key); + self.hitmap.fill_with_range( + probe, + prefix.diff_key_range().end, + self.block.len(), + ); match full_probe_key.cmp(&key) { Ordering::Less | Ordering::Equal => true, Ordering::Greater => false, } }, ) - .saturating_sub(1) // Prevent from underflowing when given is smaller than the first. + // Prevent from underflowing when given is smaller than the first. + .saturating_sub(1); + + res } /// Seeks to the restart point that the given `key` belongs to. @@ -285,6 +314,9 @@ impl BlockIterator { self.offset = offset; self.entry_len = prefix.entry_len(); self.update_restart_point(index); + + self.hitmap + .fill_with_range(self.offset, self.value_range.end, self.block.len()); } fn update_restart_point(&mut self, index: usize) { diff --git a/src/storage/src/hummock/sstable/builder.rs b/src/storage/src/hummock/sstable/builder.rs index a131017aabdae..3fdf7181d5c98 100644 --- a/src/storage/src/hummock/sstable/builder.rs +++ b/src/storage/src/hummock/sstable/builder.rs @@ -200,12 +200,13 @@ impl SstableBuilder { } if is_max_epoch(event.new_epoch) && self.monotonic_deletes.last().map_or(true, |last| { - is_max_epoch(last.new_epoch) - && last.event_key.left_user_key.table_id - == event.event_key.left_user_key.table_id + last.event_key.left_user_key.table_id != event.event_key.left_user_key.table_id + || is_max_epoch(last.new_epoch) }) { - // This range would never delete any key so we can merge it with last range. + // There are two case we shall skip the right end of delete-range. + // 1, it belongs the same table-id with the last event, and the last event is also right-end of some delete-range so we can merge them into one point. + // 2, this point does not belong the same table-id with the last event. It means that the left end of this delete-range may be dropped, so we can not add it. return; } if !is_max_epoch(event.new_epoch) { @@ -239,7 +240,6 @@ impl SstableBuilder { self.add(full_key, value).await } - /// only for test pub fn current_block_size(&self) -> usize { self.block_builder.approximate_len() } @@ -343,6 +343,12 @@ impl SstableBuilder { || !user_key(&self.raw_key).eq(user_key(&self.last_full_key)); let table_id = full_key.user_key.table_id.table_id(); let is_new_table = self.last_table_id.is_none() || self.last_table_id.unwrap() != table_id; + let current_block_size = self.current_block_size(); + let is_block_full = current_block_size >= self.options.block_capacity + || (current_block_size > self.options.block_capacity / 4 * 3 + && current_block_size + self.raw_value.len() + self.raw_key.len() + > self.options.block_capacity); + if is_new_table { assert!( could_switch_block, @@ -355,10 +361,7 @@ impl SstableBuilder { if !self.block_builder.is_empty() { self.build_block().await?; } - } else if is_new_user_key - && self.block_builder.approximate_len() >= self.options.block_capacity - && could_switch_block - { + } else if is_block_full && could_switch_block { self.build_block().await?; } self.last_table_stats.total_key_count += 1; @@ -704,6 +707,15 @@ impl SstableBuilder { data_len, block_meta.offset ) }); + + if data_len as usize > self.options.capacity * 2 { + tracing::warn!( + "WARN unexpected block size {} table {:?}", + data_len, + self.block_builder.table_id() + ); + } + self.block_builder.clear(); Ok(()) } @@ -852,14 +864,14 @@ pub(super) mod tests { .await .unwrap(); - assert_eq!(table.value().has_bloom_filter(), with_blooms); + assert_eq!(table.has_bloom_filter(), with_blooms); for i in 0..key_count { let full_key = test_key_of(i); - if table.value().has_bloom_filter() { + if table.has_bloom_filter() { let hash = Sstable::hash_for_bloom_filter(full_key.user_key.encode().as_slice(), 0); let key_ref = full_key.user_key.as_ref(); assert!( - table.value().may_match_hash( + table.may_match_hash( &(Bound::Included(key_ref), Bound::Included(key_ref)), hash ), @@ -941,9 +953,9 @@ pub(super) mod tests { let k = UserKey::for_test(TableId::new(2), table_key.as_slice()); let hash = Sstable::hash_for_bloom_filter(&k.encode(), 2); let key_ref = k.as_ref(); - assert!(table - .value() - .may_match_hash(&(Bound::Included(key_ref), Bound::Included(key_ref)), hash)); + assert!( + table.may_match_hash(&(Bound::Included(key_ref), Bound::Included(key_ref)), hash) + ); } } } diff --git a/src/storage/src/hummock/sstable/delete_range_aggregator.rs b/src/storage/src/hummock/sstable/delete_range_aggregator.rs index c28b3b14a45db..bb13d90b58377 100644 --- a/src/storage/src/hummock/sstable/delete_range_aggregator.rs +++ b/src/storage/src/hummock/sstable/delete_range_aggregator.rs @@ -181,8 +181,8 @@ impl SstableDeleteRangeIterator { /// # Panics /// This function will panic if the iterator is invalid. pub fn is_last_range(&self) -> bool { - debug_assert!(self.next_idx < self.table.value().meta.monotonic_tombstone_events.len()); - self.next_idx + 1 == self.table.value().meta.monotonic_tombstone_events.len() + debug_assert!(self.next_idx < self.table.meta.monotonic_tombstone_events.len()); + self.next_idx + 1 == self.table.meta.monotonic_tombstone_events.len() } } @@ -192,14 +192,14 @@ impl DeleteRangeIterator for SstableDeleteRangeIterator { type SeekFuture<'a> = impl Future> + 'a; fn next_extended_user_key(&self) -> PointRange<&[u8]> { - self.table.value().meta.monotonic_tombstone_events[self.next_idx] + self.table.meta.monotonic_tombstone_events[self.next_idx] .event_key .as_ref() } fn current_epoch(&self) -> HummockEpoch { if self.next_idx > 0 { - self.table.value().meta.monotonic_tombstone_events[self.next_idx - 1].new_epoch + self.table.meta.monotonic_tombstone_events[self.next_idx - 1].new_epoch } else { HummockEpoch::MAX } @@ -222,20 +222,17 @@ impl DeleteRangeIterator for SstableDeleteRangeIterator { fn seek<'a>(&'a mut self, target_user_key: UserKey<&'a [u8]>) -> Self::SeekFuture<'_> { async move { let target_extended_user_key = PointRange::from_user_key(target_user_key, false); - self.next_idx = self - .table - .value() - .meta - .monotonic_tombstone_events - .partition_point(|MonotonicDeleteEvent { event_key, .. }| { + self.next_idx = self.table.meta.monotonic_tombstone_events.partition_point( + |MonotonicDeleteEvent { event_key, .. }| { event_key.as_ref().le(&target_extended_user_key) - }); + }, + ); Ok(()) } } fn is_valid(&self) -> bool { - self.next_idx < self.table.value().meta.monotonic_tombstone_events.len() + self.next_idx < self.table.meta.monotonic_tombstone_events.len() } } @@ -491,27 +488,27 @@ mod tests { .await .unwrap(); let ret = get_min_delete_range_epoch_from_sstable( - sstable.value(), + &sstable, iterator_test_user_key_of(0).as_ref(), ); assert_eq!(ret, test_epoch(300)); let ret = get_min_delete_range_epoch_from_sstable( - sstable.value(), + &sstable, iterator_test_user_key_of(1).as_ref(), ); assert_eq!(ret, test_epoch(150)); let ret = get_min_delete_range_epoch_from_sstable( - sstable.value(), + &sstable, iterator_test_user_key_of(3).as_ref(), ); assert_eq!(ret, test_epoch(50)); let ret = get_min_delete_range_epoch_from_sstable( - sstable.value(), + &sstable, iterator_test_user_key_of(6).as_ref(), ); assert_eq!(ret, test_epoch(150)); let ret = get_min_delete_range_epoch_from_sstable( - sstable.value(), + &sstable, iterator_test_user_key_of(8).as_ref(), ); assert!(is_max_epoch(ret)); diff --git a/src/storage/src/hummock/sstable/forward_sstable_iterator.rs b/src/storage/src/hummock/sstable/forward_sstable_iterator.rs index 86a0095f7dcea..90aef85813d67 100644 --- a/src/storage/src/hummock/sstable/forward_sstable_iterator.rs +++ b/src/storage/src/hummock/sstable/forward_sstable_iterator.rs @@ -18,6 +18,7 @@ use std::sync::Arc; use await_tree::InstrumentAwait; use risingwave_hummock_sdk::key::FullKey; +use thiserror_ext::AsReport; use super::super::{HummockResult, HummockValue}; use crate::hummock::block_stream::BlockStream; @@ -75,7 +76,7 @@ impl SstableIterator { fn init_block_prefetch_range(&mut self, start_idx: usize) { self.preload_end_block_idx = 0; if let Some(bound) = self.options.must_iterated_end_user_key.as_ref() { - let block_metas = &self.sst.value().meta.block_metas; + let block_metas = &self.sst.meta.block_metas; let next_to_start_idx = start_idx + 1; if next_to_start_idx < block_metas.len() { let end_idx = match bound { @@ -113,7 +114,7 @@ impl SstableIterator { tracing::debug!( target: "events::storage::sstable::block_seek", "table iterator seek: sstable_object_id = {}, block_id = {}", - self.sst.value().id, + self.sst.id, idx, ); @@ -123,7 +124,7 @@ impl SstableIterator { tokio::task::consume_budget().await; let mut hit_cache = false; - if idx >= self.sst.value().block_count() { + if idx >= self.sst.block_count() { self.block_iter = None; return Ok(()); } @@ -131,14 +132,20 @@ impl SstableIterator { if self.preload_stream.is_none() && idx + 1 < self.preload_end_block_idx { match self .sstable_store - .prefetch_blocks(self.sst.value(), idx, self.preload_end_block_idx, - self.options.cache_policy, - &mut self.stats, + .prefetch_blocks( + &self.sst, + idx, + self.preload_end_block_idx, + self.options.cache_policy, + &mut self.stats, ) .verbose_instrument_await("prefetch_blocks") - .await { + .await + { Ok(preload_stream) => self.preload_stream = Some(preload_stream), - Err(e) => tracing::warn!("failed to create stream for prefetch data because of {:?}, fall back to block get.", e), + Err(e) => { + tracing::warn!(error = %e.as_report(), "failed to create stream for prefetch data, fall back to block get") + } } } @@ -177,7 +184,7 @@ impl SstableIterator { } if self.preload_stream.is_none() && idx + 1 < self.preload_end_block_idx { if let Err(e) = ret { - tracing::warn!("recreate stream because the connection to remote storage has closed, reason: {:?}", e); + tracing::warn!(error = %e.as_report(), "recreate stream because the connection to remote storage has closed"); if self.preload_retry_times >= self.options.max_preload_retry_times { break; } @@ -187,7 +194,7 @@ impl SstableIterator { match self .sstable_store .prefetch_blocks( - self.sst.value(), + &self.sst, idx, self.preload_end_block_idx, self.options.cache_policy, @@ -200,7 +207,7 @@ impl SstableIterator { self.preload_stream = Some(stream); } Err(e) => { - tracing::warn!("failed to recreate stream meet IO error: {:?}", e); + tracing::warn!(error = %e.as_report(), "failed to recreate stream meet IO error"); break; } } @@ -210,12 +217,7 @@ impl SstableIterator { if !hit_cache { let block = self .sstable_store - .get( - self.sst.value(), - idx, - self.options.cache_policy, - &mut self.stats, - ) + .get(&self.sst, idx, self.options.cache_policy, &mut self.stats) .await?; self.block_iter = Some(BlockIterator::new(block)); }; @@ -267,7 +269,6 @@ impl HummockIterator for SstableIterator { async fn seek<'a>(&'a mut self, key: FullKey<&'a [u8]>) -> HummockResult<()> { let block_idx = self .sst - .value() .meta .block_metas .partition_point(|block_meta| { @@ -357,7 +358,7 @@ mod tests { .await; // We should have at least 10 blocks, so that sstable iterator test could cover more code // path. - assert!(sstable.value().meta.block_metas.len() > 10); + assert!(sstable.meta.block_metas.len() > 10); inner_test_forward_iterator(sstable_store.clone(), sstable).await; } @@ -370,7 +371,7 @@ mod tests { .await; // We should have at least 10 blocks, so that sstable iterator test could cover more code // path. - assert!(sstable.value().meta.block_metas.len() > 10); + assert!(sstable.meta.block_metas.len() > 10); let mut sstable_iter = SstableIterator::create( sstable, sstable_store, diff --git a/src/storage/src/hummock/sstable/sstable_object_id_manager.rs b/src/storage/src/hummock/sstable/sstable_object_id_manager.rs index 21fe86dcaad74..c32867e45d1e8 100644 --- a/src/storage/src/hummock/sstable/sstable_object_id_manager.rs +++ b/src/storage/src/hummock/sstable/sstable_object_id_manager.rs @@ -26,6 +26,7 @@ use risingwave_pb::hummock::GetNewSstIdsRequest; use risingwave_pb::meta::heartbeat_request::extra_info::Info; use risingwave_rpc_client::{ExtraInfoSource, GrpcCompactorProxyClient, HummockMetaClient}; use sync_point::sync_point; +use thiserror_ext::AsReport; use tokio::sync::oneshot; use crate::hummock::{HummockError, HummockResult}; @@ -284,8 +285,8 @@ impl GetObjectId for SharedComapctorObjectIdManager { Ok(start_id) } Err(e) => Err(HummockError::other(format!( - "Fail to get new sst id, {}", - e + "Fail to get new sst id: {}", + e.as_report() ))), } } diff --git a/src/storage/src/hummock/sstable/xor_filter.rs b/src/storage/src/hummock/sstable/xor_filter.rs index 004d38dc9c801..8924a1a36002a 100644 --- a/src/storage/src/hummock/sstable/xor_filter.rs +++ b/src/storage/src/hummock/sstable/xor_filter.rs @@ -504,11 +504,11 @@ mod tests { .await .unwrap(); let mut stat = StoreLocalStatistic::default(); - if let XorFilter::BlockXor16(reader) = &sstable.value().filter_reader.filter { - for idx in 0..sstable.value().meta.block_metas.len() { + if let XorFilter::BlockXor16(reader) = &sstable.filter_reader.filter { + for idx in 0..sstable.meta.block_metas.len() { let resp = sstable_store .get_block_response( - sstable.value(), + &sstable, idx, CachePolicy::Fill(CachePriority::High), &mut stat, diff --git a/src/storage/src/hummock/sstable_store.rs b/src/storage/src/hummock/sstable_store.rs index 9918274b8ee91..f0cacf863fcc9 100644 --- a/src/storage/src/hummock/sstable_store.rs +++ b/src/storage/src/hummock/sstable_store.rs @@ -33,6 +33,7 @@ use risingwave_object_store::object::{ ObjectError, ObjectMetadataIter, ObjectStoreRef, ObjectStreamingUploader, }; use risingwave_pb::hummock::SstableInfo; +use thiserror_ext::AsReport; use tokio::task::JoinHandle; use tokio::time::Instant; use zstd::zstd_safe::WriteBuf; @@ -50,7 +51,7 @@ use crate::hummock::multi_builder::UploadJoinHandle; use crate::hummock::{ BlockHolder, CacheableEntry, HummockError, HummockResult, LruCache, MemoryLimiter, }; -use crate::monitor::{MemoryCollector, StoreLocalStatistic}; +use crate::monitor::{HummockStateStoreMetrics, MemoryCollector, StoreLocalStatistic}; const MAX_META_CACHE_SHARD_BITS: usize = 2; const MAX_CACHE_SHARD_BITS: usize = 6; // It means that there will be 64 shards lru-cache to avoid lock conflict. @@ -101,6 +102,7 @@ impl From for TracedCachePolicy { struct BlockCacheEventListener { data_file_cache: FileCache, + metrics: Arc, } impl LruCacheEventListener for BlockCacheEventListener { @@ -112,6 +114,10 @@ impl LruCacheEventListener for BlockCacheEventListener { sst_id: key.0, block_idx: key.1, }; + self.metrics + .block_efficiency_histogram + .with_label_values(&[&value.table_id().to_string()]) + .observe(value.efficiency()); // temporarily avoid spawn task while task drop with madsim // FYI: https://github.com/madsim-rs/madsim/issues/182 #[cfg(not(madsim))] @@ -158,6 +164,20 @@ where } } +pub struct SstableStoreConfig { + pub store: ObjectStoreRef, + pub path: String, + pub block_cache_capacity: usize, + pub meta_cache_capacity: usize, + pub high_priority_ratio: usize, + pub prefetch_buffer_capacity: usize, + pub max_prefetch_block_number: usize, + pub data_file_cache: FileCache, + pub meta_file_cache: FileCache, + pub recent_filter: Option>>, + pub state_store_metrics: Arc, +} + pub struct SstableStore { path: String, store: ObjectStoreRef, @@ -176,51 +196,43 @@ pub struct SstableStore { } impl SstableStore { - pub fn new( - store: ObjectStoreRef, - path: String, - block_cache_capacity: usize, - meta_cache_capacity: usize, - high_priority_ratio: usize, - prefetch_buffer_capacity: usize, - max_prefetch_block_number: usize, - data_file_cache: FileCache, - meta_file_cache: FileCache, - recent_filter: Option>>, - ) -> Self { + pub fn new(config: SstableStoreConfig) -> Self { // TODO: We should validate path early. Otherwise object store won't report invalid path // error until first write attempt. let mut shard_bits = MAX_META_CACHE_SHARD_BITS; - while (meta_cache_capacity >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD && shard_bits > 0 { + while (config.meta_cache_capacity >> shard_bits) < MIN_BUFFER_SIZE_PER_SHARD + && shard_bits > 0 + { shard_bits -= 1; } let block_cache_listener = Arc::new(BlockCacheEventListener { - data_file_cache: data_file_cache.clone(), + data_file_cache: config.data_file_cache.clone(), + metrics: config.state_store_metrics, }); - let meta_cache_listener = Arc::new(MetaCacheEventListener(meta_file_cache.clone())); + let meta_cache_listener = Arc::new(MetaCacheEventListener(config.meta_file_cache.clone())); Self { - path, - store, + path: config.path, + store: config.store, block_cache: BlockCache::with_event_listener( - block_cache_capacity, + config.block_cache_capacity, MAX_CACHE_SHARD_BITS, - high_priority_ratio, + config.high_priority_ratio, block_cache_listener, ), meta_cache: Arc::new(LruCache::with_event_listener( shard_bits, - meta_cache_capacity, + config.meta_cache_capacity, 0, meta_cache_listener, )), - data_file_cache, - meta_file_cache, + data_file_cache: config.data_file_cache, + meta_file_cache: config.meta_file_cache, - recent_filter, + recent_filter: config.recent_filter, prefetch_buffer_usage: Arc::new(AtomicUsize::new(0)), - prefetch_buffer_capacity, - max_prefetch_block_number, + prefetch_buffer_capacity: config.prefetch_buffer_capacity, + max_prefetch_block_number: config.max_prefetch_block_number, } } @@ -286,7 +298,7 @@ impl SstableStore { pub fn delete_cache(&self, object_id: HummockSstableObjectId) { self.meta_cache.erase(object_id, &object_id); if let Err(e) = self.meta_file_cache.remove(&object_id) { - tracing::warn!("meta file cache remove error: {}", e); + tracing::warn!(error = %e.as_report(), "meta file cache remove error"); } } @@ -561,7 +573,7 @@ impl SstableStore { pub fn clear_block_cache(&self) { self.block_cache.clear(); if let Err(e) = self.data_file_cache.clear() { - tracing::warn!("data file cache clear error: {}", e); + tracing::warn!(error = %e.as_report(), "data file cache clear error"); } } @@ -569,7 +581,7 @@ impl SstableStore { pub fn clear_meta_cache(&self) { self.meta_cache.clear(); if let Err(e) = self.meta_file_cache.clear() { - tracing::warn!("meta file cache clear error: {}", e); + tracing::warn!(error = %e.as_report(), "meta file cache clear error"); } } @@ -714,8 +726,8 @@ impl SstableStore { Ok(Err(e)) => return Err(HummockError::from(e)), Err(e) => { return Err(HummockError::other(format!( - "failed to get result, this read request may be canceled: {:?}", - e + "failed to get result, this read request may be canceled: {}", + e.as_report() ))) } }; @@ -1008,9 +1020,9 @@ impl SstableWriter for StreamingUploadWriter { } async fn finish(mut self, meta: SstableMeta) -> HummockResult { - let meta_data = Bytes::from(meta.encode_to_bytes()); + let metadata = Bytes::from(meta.encode_to_bytes()); - self.object_uploader.write_bytes(meta_data).await?; + self.object_uploader.write_bytes(metadata).await?; let join_handle = tokio::spawn(async move { let uploader_memory_usage = self.object_uploader.get_memory_usage(); let _tracker = self.tracker.map(|mut t| { @@ -1197,9 +1209,9 @@ mod tests { let mut stats = StoreLocalStatistic::default(); let holder = sstable_store.sstable(info, &mut stats).await.unwrap(); std::mem::take(&mut meta.bloom_filter); - assert_eq!(holder.value().meta, meta); + assert_eq!(holder.meta, meta); let holder = sstable_store.sstable(info, &mut stats).await.unwrap(); - assert_eq!(holder.value().meta, meta); + assert_eq!(holder.meta, meta); let mut iter = SstableIterator::new( holder, sstable_store, diff --git a/src/storage/src/hummock/store/hummock_storage.rs b/src/storage/src/hummock/store/hummock_storage.rs index 96049bd90d140..d2064ffd09dee 100644 --- a/src/storage/src/hummock/store/hummock_storage.rs +++ b/src/storage/src/hummock/store/hummock_storage.rs @@ -16,7 +16,6 @@ use std::future::Future; use std::ops::{Bound, Deref}; use std::sync::atomic::{AtomicU64, Ordering as MemOrdering}; use std::sync::Arc; -use std::time::Duration; use arc_swap::ArcSwap; use bytes::Bytes; @@ -32,7 +31,7 @@ use risingwave_pb::hummock::SstableInfo; use risingwave_rpc_client::HummockMetaClient; use tokio::sync::mpsc::{unbounded_channel, UnboundedSender}; use tokio::sync::oneshot; -use tracing::log::error; +use tracing::error; use super::local_hummock_storage::LocalHummockStorage; use super::version::{CommittedVersion, HummockVersionReader}; @@ -41,9 +40,8 @@ use crate::filter_key_extractor::{FilterKeyExtractorManager, RpcFilterKeyExtract use crate::hummock::backup_reader::{BackupReader, BackupReaderRef}; use crate::hummock::compactor::CompactorContext; use crate::hummock::event_handler::hummock_event_handler::BufferTracker; -use crate::hummock::event_handler::refiller::CacheRefillConfig; use crate::hummock::event_handler::{ - HummockEvent, HummockEventHandler, HummockVersionUpdate, ReadVersionMappingType, + HummockEvent, HummockEventHandler, HummockVersionUpdate, ReadOnlyReadVersionMapping, }; use crate::hummock::local_version::pinned_version::{start_pinned_version_worker, PinnedVersion}; use crate::hummock::observer_manager::HummockObserverNode; @@ -69,7 +67,7 @@ impl Drop for HummockStorageShutdownGuard { let _ = self .shutdown_sender .send(HummockEvent::Shutdown) - .inspect_err(|e| error!("unable to send shutdown: {:?}", e)); + .inspect_err(|e| error!(event = ?e.0, "unable to send shutdown")); } } @@ -81,6 +79,8 @@ impl Drop for HummockStorageShutdownGuard { #[derive(Clone)] pub struct HummockStorage { hummock_event_sender: UnboundedSender, + // only used in test for setting hummock version in uploader + _version_update_sender: UnboundedSender, context: CompactorContext, @@ -100,7 +100,7 @@ pub struct HummockStorage { _shutdown_guard: Arc, - read_version_mapping: Arc, + read_version_mapping: ReadOnlyReadVersionMapping, backup_reader: BackupReaderRef, @@ -153,22 +153,22 @@ impl HummockStorage { .await .map_err(HummockError::read_backup_error)?; let write_limiter = Arc::new(WriteLimiter::default()); - let (event_tx, mut event_rx) = unbounded_channel(); + let (version_update_tx, mut version_update_rx) = unbounded_channel(); let observer_manager = ObserverManager::new( notification_client, HummockObserverNode::new( filter_key_extractor_manager.clone(), backup_reader.clone(), - event_tx.clone(), + version_update_tx.clone(), write_limiter.clone(), ), ) .await; observer_manager.start().await; - let hummock_version = match event_rx.recv().await { - Some(HummockEvent::VersionUpdate(HummockVersionUpdate::PinnedVersion(version))) => version, + let hummock_version = match version_update_rx.recv().await { + Some(HummockVersionUpdate::PinnedVersion(version)) => version, _ => unreachable!("the hummock observer manager is the first one to take the event tx. Should be full hummock version") }; @@ -191,26 +191,16 @@ impl HummockStorage { let seal_epoch = Arc::new(AtomicU64::new(pinned_version.max_committed_epoch())); let min_current_epoch = Arc::new(AtomicU64::new(pinned_version.max_committed_epoch())); let hummock_event_handler = HummockEventHandler::new( - event_tx.clone(), - event_rx, + version_update_rx, pinned_version, compactor_context.clone(), filter_key_extractor_manager.clone(), sstable_object_id_manager.clone(), state_store_metrics.clone(), - CacheRefillConfig { - timeout: Duration::from_millis(options.cache_refill_timeout_ms), - data_refill_levels: options - .cache_refill_data_refill_levels - .iter() - .copied() - .collect(), - concurrency: options.cache_refill_concurrency, - unit: options.cache_refill_unit, - threshold: options.cache_refill_threshold, - }, ); + let event_tx = hummock_event_handler.event_sender(); + let instance = Self { context: compactor_context, filter_key_extractor_manager: filter_key_extractor_manager.clone(), @@ -219,6 +209,7 @@ impl HummockStorage { version_update_notifier_tx: hummock_event_handler.version_update_notifier_tx(), seal_epoch, hummock_event_sender: event_tx.clone(), + _version_update_sender: version_update_tx, pinned_version: hummock_event_handler.pinned_version(), hummock_version_reader: HummockVersionReader::new( sstable_store, @@ -480,10 +471,10 @@ impl StateStore for HummockStorage { StoreLocalStatistic::flush_all(); } - async fn clear_shared_buffer(&self) -> StorageResult<()> { + async fn clear_shared_buffer(&self, prev_epoch: u64) { let (tx, rx) = oneshot::channel(); self.hummock_event_sender - .send(HummockEvent::Clear(tx)) + .send(HummockEvent::Clear(tx, prev_epoch)) .expect("should send success"); rx.await.expect("should wait success"); @@ -491,8 +482,6 @@ impl StateStore for HummockStorage { self.min_current_epoch .store(HummockEpoch::MAX, MemOrdering::SeqCst); self.seal_epoch.store(epoch, MemOrdering::SeqCst); - - Ok(()) } fn new_local(&self, option: NewLocalOptions) -> impl Future + Send + '_ { @@ -540,13 +529,12 @@ impl HummockStorage { } /// Used in the compaction test tool + #[cfg(any(test, feature = "test"))] pub async fn update_version_and_wait(&self, version: HummockVersion) { use tokio::task::yield_now; let version_id = version.id; - self.hummock_event_sender - .send(HummockEvent::VersionUpdate( - HummockVersionUpdate::PinnedVersion(version), - )) + self._version_update_sender + .send(HummockVersionUpdate::PinnedVersion(version)) .unwrap(); loop { if self.pinned_version.load().id() >= version_id { diff --git a/src/storage/src/hummock/store/local_hummock_storage.rs b/src/storage/src/hummock/store/local_hummock_storage.rs index a9a1899d8e552..b4e4f041b02f3 100644 --- a/src/storage/src/hummock/store/local_hummock_storage.rs +++ b/src/storage/src/hummock/store/local_hummock_storage.rs @@ -18,7 +18,7 @@ use std::sync::Arc; use await_tree::InstrumentAwait; use bytes::Bytes; -use parking_lot::RwLock; +use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::{TableId, TableOption}; use risingwave_common::util::epoch::MAX_SPILL_TIMES; use risingwave_hummock_sdk::key::{is_empty_key_range, TableKey, TableKeyRange}; @@ -26,20 +26,20 @@ use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch}; use tokio::sync::mpsc; use tracing::{warn, Instrument}; -use super::version::{HummockReadVersion, StagingData, VersionUpdate}; +use super::version::{StagingData, VersionUpdate}; use crate::error::StorageResult; -use crate::hummock::event_handler::{HummockEvent, LocalInstanceGuard}; +use crate::hummock::event_handler::{HummockEvent, HummockReadVersionRef, LocalInstanceGuard}; use crate::hummock::iterator::{ - ConcatIteratorInner, Forward, HummockIteratorUnion, OrderedMergeIteratorInner, - SkipWatermarkIterator, UnorderedMergeIteratorInner, UserIterator, + ConcatIteratorInner, Forward, HummockIteratorUnion, MergeIterator, SkipWatermarkIterator, + UserIterator, }; use crate::hummock::shared_buffer::shared_buffer_batch::{ SharedBufferBatch, SharedBufferBatchIterator, }; use crate::hummock::store::version::{read_filter_for_local, HummockVersionReader}; use crate::hummock::utils::{ - cmp_delete_range_left_bounds, do_delete_sanity_check, do_insert_sanity_check, - do_update_sanity_check, filter_with_delete_range, wait_for_epoch, ENABLE_SANITY_CHECK, + do_delete_sanity_check, do_insert_sanity_check, do_update_sanity_check, wait_for_epoch, + ENABLE_SANITY_CHECK, }; use crate::hummock::write_limiter::WriteLimiterRef; use crate::hummock::{MemoryLimiter, SstableIterator}; @@ -64,7 +64,7 @@ pub struct LocalHummockStorage { instance_guard: LocalInstanceGuard, /// Read handle. - read_version: Arc>, + read_version: HummockReadVersionRef, /// This indicates that this `LocalHummockStorage` replicates another `LocalHummockStorage`. /// It's used by executors in different CNs to synchronize states. @@ -284,17 +284,10 @@ impl LocalStateStore for LocalHummockStorage { Ok(()) } - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> StorageResult { - debug_assert!(delete_ranges - .iter() - .map(|(key, _)| key) - .is_sorted_by(|a, b| Some(cmp_delete_range_left_bounds(a.as_ref(), b.as_ref())))); + async fn flush(&mut self) -> StorageResult { let buffer = self.mem_table.drain().into_parts(); let mut kv_pairs = Vec::with_capacity(buffer.len()); - for (key, key_op) in filter_with_delete_range(buffer.into_iter(), delete_ranges.iter()) { + for (key, key_op) in buffer { match key_op { // Currently, some executors do not strictly comply with these semantics. As // a workaround you may call disable the check by initializing the @@ -349,7 +342,6 @@ impl LocalStateStore for LocalHummockStorage { } self.flush_inner( kv_pairs, - delete_ranges, WriteOptions { epoch: self.epoch(), table_id: self.table_id, @@ -364,7 +356,7 @@ impl LocalStateStore for LocalHummockStorage { { if self.spill_offset < MAX_SPILL_TIMES { let table_id_label = self.table_id.table_id().to_string(); - self.flush(vec![]).await?; + self.flush().await?; self.stats .mem_table_spill_counts .with_label_values(&[table_id_label.as_str()]) @@ -395,12 +387,24 @@ impl LocalStateStore for LocalHummockStorage { "local state store of table id {:?} is init for more than once", self.table_id ); + let prev_vnodes = self + .read_version + .write() + .update_vnode_bitmap(options.vnodes); + assert!( + prev_vnodes.is_none(), + "Vnode bitmap should be empty during init" + ); Ok(()) } fn seal_current_epoch(&mut self, next_epoch: u64, mut opts: SealCurrentEpochOptions) { assert!(!self.is_dirty()); + if let Some(new_level) = &opts.switch_op_consistency_level { + self.mem_table.op_consistency_level.update(new_level); + self.op_consistency_level.update(new_level); + } let prev_epoch = self .epoch .replace(next_epoch) @@ -432,13 +436,21 @@ impl LocalStateStore for LocalHummockStorage { }) .expect("should be able to send") } + + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { + let mut read_version = self.read_version.write(); + assert!(read_version.staging().is_empty(), "There is uncommitted staging data in read version table_id {:?} instance_id {:?} on vnode bitmap update", + self.table_id(), self.instance_id() + ); + let prev_vnodes = read_version.update_vnode_bitmap(vnodes); + prev_vnodes.expect("Previous vnode bitmap should not be none") + } } impl LocalHummockStorage { async fn flush_inner( &mut self, kv_pairs: Vec<(TableKey, StorageValue)>, - delete_ranges: Vec<(Bound, Bound)>, write_options: WriteOptions, ) -> StorageResult { let epoch = write_options.epoch; @@ -455,10 +467,9 @@ impl LocalHummockStorage { .with_label_values(&[table_id_label.as_str()]) .start_timer(); - let imm_size = if !kv_pairs.is_empty() || !delete_ranges.is_empty() { + let imm_size = if !kv_pairs.is_empty() { let sorted_items = SharedBufferBatch::build_shared_buffer_item_batches(kv_pairs); - let size = SharedBufferBatch::measure_batch_size(&sorted_items) - + SharedBufferBatch::measure_delete_range_size(&delete_ranges); + let size = SharedBufferBatch::measure_batch_size(&sorted_items); self.write_limiter.wait_permission(self.table_id).await; let limiter = self.memory_limiter.as_ref(); let tracker = if let Some(tracker) = limiter.try_require_memory(size as u64) { @@ -490,9 +501,8 @@ impl LocalHummockStorage { self.spill_offset, sorted_items, size, - delete_ranges, table_id, - Some(instance_id), + instance_id, Some(tracker), ); self.spill_offset += 1; @@ -524,7 +534,7 @@ impl LocalHummockStorage { #[allow(clippy::too_many_arguments)] pub fn new( instance_guard: LocalInstanceGuard, - read_version: Arc>, + read_version: HummockReadVersionRef, hummock_version_reader: HummockVersionReader, event_sender: mpsc::UnboundedSender, memory_limiter: Arc, @@ -555,7 +565,7 @@ impl LocalHummockStorage { } /// See `HummockReadVersion::update` for more details. - pub fn read_version(&self) -> Arc> { + pub fn read_version(&self) -> HummockReadVersionRef { self.read_version.clone() } @@ -568,11 +578,11 @@ impl LocalHummockStorage { } } -pub type StagingDataIterator = OrderedMergeIteratorInner< +pub type StagingDataIterator = MergeIterator< HummockIteratorUnion, SstableIterator>, >; pub type HummockStorageIteratorPayloadInner<'a> = SkipWatermarkIterator< - UnorderedMergeIteratorInner< + MergeIterator< HummockIteratorUnion< Forward, StagingDataIterator, diff --git a/src/storage/src/hummock/store/version.rs b/src/storage/src/hummock/store/version.rs index c864c02b1e29e..513451b848521 100644 --- a/src/storage/src/hummock/store/version.rs +++ b/src/storage/src/hummock/store/version.rs @@ -16,12 +16,14 @@ use std::cmp::Ordering; use std::collections::vec_deque::VecDeque; use std::collections::HashSet; use std::iter::once; +use std::ops::Bound::Included; use std::sync::Arc; use await_tree::InstrumentAwait; use bytes::Bytes; use itertools::Itertools; use parking_lot::RwLock; +use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::TableId; use risingwave_common::util::epoch::MAX_SPILL_TIMES; use risingwave_hummock_sdk::key::{ @@ -39,9 +41,10 @@ use tracing::Instrument; use super::StagingDataIterator; use crate::error::StorageResult; +use crate::hummock::event_handler::HummockReadVersionRef; use crate::hummock::iterator::{ - ConcatIterator, ForwardMergeRangeIterator, HummockIteratorUnion, OrderedMergeIteratorInner, - SkipWatermarkIterator, UnorderedMergeIteratorInner, UserIterator, + ConcatIterator, ForwardMergeRangeIterator, HummockIteratorUnion, MergeIterator, + SkipWatermarkIterator, UserIterator, }; use crate::hummock::local_version::pinned_version::PinnedVersion; use crate::hummock::sstable::SstableIteratorReadOptions; @@ -116,7 +119,7 @@ impl StagingSstableInfo { #[derive(Clone)] pub enum StagingData { ImmMem(ImmutableMemtable), - MergedImmMem(ImmutableMemtable), + MergedImmMem(ImmutableMemtable, Vec), Sst(StagingSstableInfo), } @@ -139,10 +142,6 @@ pub struct StagingVersion { // imm of smaller batch id may be added later than one with greater batch id pub imm: VecDeque, - // Separate queue for merged imm to ease the management of imm and merged imm. - // Newer merged imm comes first - pub merged_imm: VecDeque, - // newer data comes first pub sst: VecDeque, } @@ -162,20 +161,16 @@ impl StagingVersion { let (ref left, ref right) = table_key_range; let left = left.as_ref().map(|key| TableKey(key.0.as_ref())); let right = right.as_ref().map(|key| TableKey(key.0.as_ref())); - let overlapped_imms = self - .imm - .iter() - .chain(self.merged_imm.iter()) - .filter(move |imm| { - // retain imm which is overlapped with (min_epoch_exclusive, max_epoch_inclusive] - imm.min_epoch() <= max_epoch_inclusive - && imm.table_id == table_id - && range_overlap( - &(left, right), - &imm.start_table_key(), - imm.end_table_key().as_ref(), - ) - }); + let overlapped_imms = self.imm.iter().filter(move |imm| { + // retain imm which is overlapped with (min_epoch_exclusive, max_epoch_inclusive] + imm.min_epoch() <= max_epoch_inclusive + && imm.table_id == table_id + && range_overlap( + &(left, right), + &imm.start_table_key(), + Included(&imm.end_table_key()), + ) + }); // TODO: Remove duplicate sst based on sst id let overlapped_ssts = self @@ -198,6 +193,10 @@ impl StagingVersion { }); (overlapped_imms, overlapped_ssts) } + + pub fn is_empty(&self) -> bool { + self.imm.is_empty() && self.sst.is_empty() + } } #[derive(Clone)] @@ -218,6 +217,10 @@ pub struct HummockReadVersion { is_replicated: bool, table_watermarks: Option, + + // Vnode bitmap corresponding to the read version + // It will be initialized after local state store init + vnodes: Option>, } impl HummockReadVersion { @@ -238,13 +241,13 @@ impl HummockReadVersion { .cloned(), staging: StagingVersion { imm: VecDeque::default(), - merged_imm: Default::default(), sst: VecDeque::default(), }, committed: committed_version, is_replicated, + vnodes: None, } } @@ -252,6 +255,10 @@ impl HummockReadVersion { Self::new_with_replication_option(table_id, committed_version, false) } + pub fn table_id(&self) -> TableId { + self.table_id + } + /// Updates the read version with `VersionUpdate`. /// There will be three data types to be processed /// `VersionUpdate::Staging` @@ -274,12 +281,8 @@ impl HummockReadVersion { self.staging.imm.push_front(imm) } - StagingData::MergedImmMem(merged_imm) => { - if let Some(item) = self.staging.merged_imm.front() { - // check batch_id order from newest to old - debug_assert!(item.batch_id() < merged_imm.batch_id()); - } - self.add_merged_imm(merged_imm); + StagingData::MergedImmMem(merged_imm, imm_ids) => { + self.add_merged_imm(merged_imm, imm_ids); } StagingData::Sst(staging_sst) => { // The following properties must be ensured: @@ -296,18 +299,12 @@ impl HummockReadVersion { .staging .imm .iter() - .chain(self.staging.merged_imm.iter()) .rev() .is_sorted_by_key(|imm| imm.batch_id())); // Calculate intersection - let staging_imm_ids_from_imms: HashSet = self - .staging - .imm - .iter() - .chain(self.staging.merged_imm.iter()) - .map(|imm| imm.batch_id()) - .collect(); + let staging_imm_ids_from_imms: HashSet = + self.staging.imm.iter().map(|imm| imm.batch_id()).collect(); // intersected batch_id order from oldest to newest let intersect_imm_ids = staging_sst @@ -322,21 +319,12 @@ impl HummockReadVersion { // Check 2) debug_assert!(check_subset_preserve_order( intersect_imm_ids.iter().copied(), - self.staging - .imm - .iter() - .chain(self.staging.merged_imm.iter()) - .map(|imm| imm.batch_id()) - .rev(), + self.staging.imm.iter().map(|imm| imm.batch_id()).rev(), )); // Check 3) and replace imms with a staging sst for imm_id in &intersect_imm_ids { - if let Some(merged_imm) = self.staging.merged_imm.back() { - if *imm_id == merged_imm.batch_id() { - self.staging.merged_imm.pop_back(); - } - } else if let Some(imm) = self.staging.imm.back() { + if let Some(imm) = self.staging.imm.back() { if *imm_id == imm.batch_id() { self.staging.imm.pop_back(); } @@ -348,24 +336,16 @@ impl HummockReadVersion { .map(|imm| imm.batch_id()) .collect_vec(); - let merged_imm_ids = self - .staging - .merged_imm - .iter() - .map(|imm| imm.batch_id()) - .collect_vec(); unreachable!( "should not reach here staging_sst.size {}, staging_sst.imm_ids {:?}, staging_sst.epochs {:?}, local_imm_ids {:?}, - merged_imm_ids {:?}, intersect_imm_ids {:?}", staging_sst.imm_size, staging_sst.imm_ids, staging_sst.epochs, local_imm_ids, - merged_imm_ids, intersect_imm_ids, ); } @@ -389,10 +369,6 @@ impl HummockReadVersion { .imm .retain(|imm| imm.min_epoch() > max_committed_epoch); - self.staging - .merged_imm - .retain(|merged_imm| merged_imm.min_epoch() > max_committed_epoch); - self.staging.sst.retain(|sst| { sst.epochs.first().expect("epochs not empty") > &max_committed_epoch }); @@ -444,56 +420,106 @@ impl HummockReadVersion { } } - pub fn clear_uncommitted(&mut self) { - self.staging.imm.clear(); - self.staging.merged_imm.clear(); - self.staging.sst.clear(); - self.table_watermarks = self - .committed - .table_watermark_index() - .get(&self.table_id) - .cloned() - } - - pub fn add_merged_imm(&mut self, merged_imm: ImmutableMemtable) { - let staging_imm_count = self.staging.imm.len(); - let merged_imm_ids = merged_imm.get_imm_ids(); - - #[cfg(debug_assertions)] - { - // check the suffix `merged_imm_ids.len()` imms in staging.imm are the same as - // `merged_imm_ids` - let diff = staging_imm_count - merged_imm_ids.len(); - let mut count: usize = 0; - for (i, imm) in self.staging.imm.iter().skip(diff).enumerate() { - count += 1; - assert_eq!( - imm.batch_id(), - merged_imm_ids[i], - "merged_imm_ids: {:?}", - merged_imm_ids - ); + /// `imm_ids` is the list of imm ids that are merged into this batch + /// This field is immutable. Larger imm id at the front. + pub fn add_merged_imm(&mut self, merged_imm: ImmutableMemtable, imm_ids: Vec) { + assert!(imm_ids.iter().rev().is_sorted()); + let min_imm_id = *imm_ids.last().expect("non-empty"); + + let back = self.staging.imm.back().expect("should not be empty"); + + // pop and save imms that are written earlier than the oldest imm if there is any + let earlier_imms = if back.batch_id() < min_imm_id { + let mut earlier_imms = VecDeque::with_capacity(self.staging.imm.len()); + loop { + let batch_id = self + .staging + .imm + .back() + .expect("should not be empty") + .batch_id(); + match batch_id.cmp(&min_imm_id) { + Ordering::Less => { + let imm = self.staging.imm.pop_back().unwrap(); + earlier_imms.push_front(imm); + } + Ordering::Equal => { + break; + } + Ordering::Greater => { + let remaining_staging_imm_ids = self + .staging + .imm + .iter() + .map(|imm| imm.batch_id()) + .collect_vec(); + let earlier_imm_ids = + earlier_imms.iter().map(|imm| imm.batch_id()).collect_vec(); + + unreachable!( + "must have break in equal: {:?} {:?} {:?}", + remaining_staging_imm_ids, earlier_imm_ids, imm_ids + ) + } + } } - assert_eq!(count, merged_imm_ids.len()); + Some(earlier_imms) + } else { + assert_eq!( + back.batch_id(), + min_imm_id, + "{:?} {:?}", + { + self.staging + .imm + .iter() + .map(|imm| imm.batch_id()) + .collect_vec() + }, + imm_ids + ); + None + }; + + // iter from smaller imm and take the older imm at the back. + for imm_id in imm_ids.iter().rev() { + let imm = self.staging.imm.pop_back().expect("should exist"); + assert_eq!( + imm.batch_id(), + *imm_id, + "{:?} {:?} {}", + { + self.staging + .imm + .iter() + .map(|imm| imm.batch_id()) + .collect_vec() + }, + imm_ids, + imm_id, + ); } - self.staging - .imm - .truncate(staging_imm_count - merged_imm_ids.len()); - // add the newly merged imm into front - self.staging.merged_imm.push_front(merged_imm); + self.staging.imm.push_back(merged_imm); + if let Some(earlier_imms) = earlier_imms { + self.staging.imm.extend(earlier_imms); + } } pub fn is_replicated(&self) -> bool { self.is_replicated } + + pub fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Option> { + self.vnodes.replace(vnodes) + } } pub fn read_filter_for_batch( epoch: HummockEpoch, // for check table_id: TableId, mut key_range: TableKeyRange, - read_version_vec: Vec>>, + read_version_vec: Vec, ) -> StorageResult<(TableKeyRange, ReadVersionTuple)> { assert!(!read_version_vec.is_empty()); let mut staging_vec = Vec::with_capacity(read_version_vec.len()); @@ -537,22 +563,29 @@ pub fn read_filter_for_batch( let mut imm_vec = Vec::default(); let mut sst_vec = Vec::default(); + let mut seen_imm_ids = HashSet::new(); + let mut seen_sst_ids = HashSet::new(); // only filter the staging data that epoch greater than max_mce to avoid data duplication let (min_epoch, max_epoch) = (max_mce_version.max_committed_epoch(), epoch); // prune imm and sst with max_mce for (staging_imms, staging_ssts) in staging_vec { - imm_vec.extend( - staging_imms - .into_iter() - .filter(|imm| imm.min_epoch() > min_epoch && imm.min_epoch() <= max_epoch), - ); + imm_vec.extend(staging_imms.into_iter().filter(|imm| { + // There shouldn't be duplicated IMMs because merge imm only operates on a single shard. + assert!(seen_imm_ids.insert(imm.batch_id())); + imm.min_epoch() > min_epoch && imm.min_epoch() <= max_epoch + })); sst_vec.extend(staging_ssts.into_iter().filter(|staging_sst| { assert!( staging_sst.get_max_epoch() <= min_epoch || staging_sst.get_min_epoch() > min_epoch ); - staging_sst.min_epoch > min_epoch + // Dedup staging SSTs in different shard. Duplicates can happen in the following case: + // - Table 1 Shard 1 produces IMM 1 + // - Table 1 Shard 2 produces IMM 2 + // - IMM 1 and IMM 2 are compacted into SST 1 as a Staging SST + // - SST 1 is added to both Shard 1's and Shard 2's read version + staging_sst.min_epoch > min_epoch && seen_sst_ids.insert(staging_sst.object_id) })); } @@ -685,6 +718,8 @@ impl HummockVersionReader { Sstable::hash_for_bloom_filter(dist_key.as_ref(), read_options.table_id.table_id()) }); + // Here epoch passed in is pure epoch, and we will seek the constructed `full_key` later. + // Therefore, it is necessary to construct the `full_key` with `MAX_SPILL_TIMES`, otherwise, the iterator might skip keys with spill offset greater than 0. let full_key = FullKey::new_with_gap_epoch( read_options.table_id, TableKey(table_key.clone()), @@ -846,9 +881,6 @@ impl HummockVersionReader { let mut delete_range_iter = ForwardMergeRangeIterator::new(epoch); local_stats.staging_imm_iter_count = imms.len() as u64; for imm in imms { - if imm.has_range_tombstone() && !read_options.ignore_range_tombstone { - delete_range_iter.add_batch_iter(imm.delete_range_iter()); - } staging_iters.push(HummockIteratorUnion::First(imm.into_forward_iter())); } @@ -880,11 +912,7 @@ impl HummockVersionReader { .instrument(tracing::trace_span!("get_sstable")) .await?; - if !table_holder - .value() - .meta - .monotonic_tombstone_events - .is_empty() + if !table_holder.meta.monotonic_tombstone_events.is_empty() && !read_options.ignore_range_tombstone { delete_range_iter @@ -892,7 +920,7 @@ impl HummockVersionReader { } if let Some(prefix_hash) = bloom_filter_prefix_hash.as_ref() { if !hit_sstable_bloom_filter( - table_holder.value(), + &table_holder, &user_key_range_ref, *prefix_hash, &mut local_stats, @@ -909,7 +937,7 @@ impl HummockVersionReader { ))); } local_stats.staging_sst_iter_count = staging_sst_iter_count; - let staging_iter: StagingDataIterator = OrderedMergeIteratorInner::new(staging_iters); + let staging_iter: StagingDataIterator = MergeIterator::new(staging_iters); let mut non_overlapping_iters = Vec::new(); let mut overlapping_iters = Vec::new(); @@ -962,7 +990,7 @@ impl HummockVersionReader { .sstable(&sstables[0], &mut local_stats) .instrument(tracing::trace_span!("get_sstable")) .await?; - if !sstable.value().meta.monotonic_tombstone_events.is_empty() + if !sstable.meta.monotonic_tombstone_events.is_empty() && !read_options.ignore_range_tombstone { delete_range_iter @@ -970,7 +998,7 @@ impl HummockVersionReader { } if let Some(dist_hash) = bloom_filter_prefix_hash.as_ref() { if !hit_sstable_bloom_filter( - sstable.value(), + &sstable, &user_key_range_ref, *dist_hash, &mut local_stats, @@ -1007,8 +1035,8 @@ impl HummockVersionReader { .sstable(sstable_info, &mut local_stats) .instrument(tracing::trace_span!("get_sstable")) .await?; - assert_eq!(sstable_info.get_object_id(), sstable.value().id); - if !sstable.value().meta.monotonic_tombstone_events.is_empty() + assert_eq!(sstable_info.get_object_id(), sstable.id); + if !sstable.meta.monotonic_tombstone_events.is_empty() && !read_options.ignore_range_tombstone { delete_range_iter @@ -1016,7 +1044,7 @@ impl HummockVersionReader { } if let Some(dist_hash) = bloom_filter_prefix_hash.as_ref() { if !hit_sstable_bloom_filter( - sstable.value(), + &sstable, &user_key_range_ref, *dist_hash, &mut local_stats, @@ -1043,7 +1071,7 @@ impl HummockVersionReader { } // 3. build user_iterator - let merge_iter = UnorderedMergeIteratorInner::new( + let merge_iter = MergeIterator::new( once(HummockIteratorUnion::First(staging_iter)) .chain( overlapping_iters @@ -1167,7 +1195,7 @@ impl HummockVersionReader { self.sstable_store .sstable(local_sst, &mut stats_guard.local_stats) .await? - .value(), + .as_ref(), &user_key_range_ref, bloom_filter_prefix_hash, &mut stats_guard.local_stats, @@ -1194,7 +1222,7 @@ impl HummockVersionReader { self.sstable_store .sstable(sstable_info, &mut stats_guard.local_stats) .await? - .value(), + .as_ref(), &user_key_range_ref, bloom_filter_prefix_hash, &mut stats_guard.local_stats, @@ -1213,7 +1241,7 @@ impl HummockVersionReader { self.sstable_store .sstable(table_info, &mut stats_guard.local_stats) .await? - .value(), + .as_ref(), &user_key_range_ref, bloom_filter_prefix_hash, &mut stats_guard.local_stats, diff --git a/src/storage/src/hummock/test_utils.rs b/src/storage/src/hummock/test_utils.rs index 9bc388fb90b7b..3bef02f7d7298 100644 --- a/src/storage/src/hummock/test_utils.rs +++ b/src/storage/src/hummock/test_utils.rs @@ -22,7 +22,6 @@ use itertools::Itertools; use risingwave_common::cache::CachePriority; use risingwave_common::catalog::TableId; use risingwave_common::hash::VirtualNode; -use risingwave_common::must_match; use risingwave_common::util::epoch::test_epoch; use risingwave_hummock_sdk::key::{FullKey, PointRange, TableKey, UserKey}; use risingwave_hummock_sdk::{EpochWithGap, HummockEpoch, HummockSstableObjectId}; @@ -94,17 +93,14 @@ pub fn gen_dummy_sst_info( epoch: HummockEpoch, ) -> SstableInfo { let mut min_table_key: Vec = batches[0].start_table_key().to_vec(); - let mut max_table_key: Vec = - must_match!(batches[0].end_table_key(), Bound::Included(table_key) => table_key.to_vec()); + let mut max_table_key: Vec = batches[0].end_table_key().to_vec(); let mut file_size = 0; for batch in batches.iter().skip(1) { if min_table_key.as_slice() > *batch.start_table_key() { min_table_key = batch.start_table_key().to_vec(); } - if max_table_key.as_slice() - < must_match!(batch.end_table_key(), Bound::Included(table_key) => *table_key) - { - max_table_key = must_match!(batch.end_table_key(), Bound::Included(table_key) => table_key.to_vec()); + if max_table_key.as_slice() < *batch.end_table_key() { + max_table_key = batch.end_table_key().to_vec(); } file_size += batch.size() as u64; } @@ -117,8 +113,10 @@ pub fn gen_dummy_sst_info( right_exclusive: false, }), file_size, - table_ids: vec![], + table_ids: vec![table_id.table_id], uncompressed_file_size: file_size, + min_epoch: epoch, + max_epoch: epoch, ..Default::default() } } @@ -393,6 +391,7 @@ pub fn create_small_table_cache() -> Arc, Bound)>, ) { - let size = SharedBufferBatch::measure_delete_range_size(&delete_ranges); - let batch = SharedBufferBatch::build_shared_buffer_batch( - test_epoch(epoch), - 0, - vec![], - size, - delete_ranges, - table_id, - None, - None, - ); - self.iter.add_batch_iter(batch.delete_range_iter()); + self.iter + .add_batch_iter(SharedBufferDeleteRangeIterator::new( + test_epoch(epoch), + table_id, + delete_ranges, + )); } pub fn build_for_compaction(self) -> CompactionDeleteRangeIterator { diff --git a/src/storage/src/hummock/utils.rs b/src/storage/src/hummock/utils.rs index 0b4700500f40b..09c4a7eef0e48 100644 --- a/src/storage/src/hummock/utils.rs +++ b/src/storage/src/hummock/utils.rs @@ -497,7 +497,7 @@ pub fn cmp_delete_range_left_bounds(a: Bound<&Bytes>, b: Bound<&Bytes>) -> Order } } -fn validate_delete_range(left: &Bound, right: &Bound) -> bool { +pub(crate) fn validate_delete_range(left: &Bound, right: &Bound) -> bool { match (left, right) { // only right bound of delete range can be `Unbounded` (Unbounded, _) => unreachable!(), @@ -509,6 +509,7 @@ fn validate_delete_range(left: &Bound, right: &Bound) -> bool { } } +#[expect(dead_code)] pub(crate) fn filter_with_delete_range<'a>( kv_iter: impl Iterator, KeyOp)> + 'a, mut delete_ranges_iter: impl Iterator, Bound)> + 'a, @@ -578,7 +579,7 @@ pub(crate) async fn wait_for_epoch( } loop { match tokio::time::timeout(Duration::from_secs(30), receiver.changed()).await { - Err(elapsed) => { + Err(_) => { // The reason that we need to retry here is batch scan in // chain/rearrange_chain is waiting for an // uncommitted epoch carried by the CreateMV barrier, which @@ -589,9 +590,8 @@ pub(crate) async fn wait_for_epoch( // CN with the same distribution as the upstream MV. // See #3845 for more details. tracing::warn!( - "wait_epoch {:?} timeout when waiting for version update elapsed {:?}s", - wait_epoch, - elapsed + epoch = wait_epoch, + "wait_epoch timeout when waiting for version update", ); continue; } diff --git a/src/storage/src/hummock/vacuum.rs b/src/storage/src/hummock/vacuum.rs index 0a5bbf1445417..5242a6eae0784 100644 --- a/src/storage/src/hummock/vacuum.rs +++ b/src/storage/src/hummock/vacuum.rs @@ -21,6 +21,7 @@ use risingwave_hummock_sdk::HummockSstableObjectId; use risingwave_object_store::object::ObjectMetadataIter; use risingwave_pb::hummock::{FullScanTask, VacuumTask}; use risingwave_rpc_client::HummockMetaClient; +use thiserror_ext::AsReport; use super::{HummockError, HummockResult}; use crate::hummock::{SstableStore, SstableStoreRef}; @@ -48,7 +49,7 @@ impl Vacuum { tracing::info!("Finished vacuuming SSTs"); } Err(e) => { - tracing::warn!("Failed to report vacuum task: {:#?}", e); + tracing::warn!(error = %e.as_report(), "Failed to report vacuum task"); return false; } } @@ -128,7 +129,7 @@ impl Vacuum { tracing::info!("Finished full scan SSTs"); } Err(e) => { - tracing::warn!("Failed to report full scan task: {:#?}", e); + tracing::warn!(error = %e.as_report(), "Failed to report full scan task"); return false; } } diff --git a/src/storage/src/lib.rs b/src/storage/src/lib.rs index 8bf78347d1803..8e6efa63e3545 100644 --- a/src/storage/src/lib.rs +++ b/src/storage/src/lib.rs @@ -40,6 +40,8 @@ #![feature(associated_type_bounds)] #![feature(exclusive_range_pattern)] #![feature(impl_trait_in_assoc_type)] +#![feature(maybe_uninit_uninit_array)] +#![feature(maybe_uninit_array_assume_init)] pub mod hummock; pub mod memory; diff --git a/src/storage/src/mem_table.rs b/src/storage/src/mem_table.rs index aed84ccf1e2ed..ffaf127a13f8d 100644 --- a/src/storage/src/mem_table.rs +++ b/src/storage/src/mem_table.rs @@ -17,26 +17,28 @@ use std::collections::btree_map::Entry; use std::collections::BTreeMap; use std::future::Future; use std::ops::Bound::{Excluded, Included, Unbounded}; -use std::ops::{Bound, RangeBounds}; +use std::ops::RangeBounds; +use std::sync::Arc; use bytes::Bytes; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; use itertools::Itertools; +use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::{TableId, TableOption}; use risingwave_common::estimate_size::{EstimateSize, KvSize}; use risingwave_common::hash::VnodeBitmapExt; use risingwave_hummock_sdk::key::{prefixed_range_with_vnode, FullKey, TableKey, TableKeyRange}; use risingwave_hummock_sdk::table_watermark::WatermarkDirection; use thiserror::Error; +use thiserror_ext::AsReport; use tracing::error; use crate::error::{StorageError, StorageResult}; use crate::hummock::iterator::{FromRustIterator, RustIteratorBuilder}; use crate::hummock::shared_buffer::shared_buffer_batch::{SharedBufferBatch, SharedBufferBatchId}; use crate::hummock::utils::{ - cmp_delete_range_left_bounds, do_delete_sanity_check, do_insert_sanity_check, - do_update_sanity_check, filter_with_delete_range, ENABLE_SANITY_CHECK, + do_delete_sanity_check, do_insert_sanity_check, do_update_sanity_check, ENABLE_SANITY_CHECK, }; use crate::hummock::value::HummockValue; use crate::row_serde::value_serde::ValueRowSerde; @@ -434,6 +436,7 @@ pub struct MemtableLocalStateStore { table_id: TableId, op_consistency_level: OpConsistencyLevel, table_option: TableOption, + vnodes: Option>, } impl MemtableLocalStateStore { @@ -445,6 +448,7 @@ impl MemtableLocalStateStore { table_id: option.table_id, op_consistency_level: option.op_consistency_level, table_option: option.table_option, + vnodes: None, } } @@ -516,17 +520,10 @@ impl LocalStateStore for MemtableLocalState Ok(self.mem_table.delete(key, old_val)?) } - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> StorageResult { - debug_assert!(delete_ranges - .iter() - .map(|(key, _)| key) - .is_sorted_by(|a, b| Some(cmp_delete_range_left_bounds(a.as_ref(), b.as_ref())))); + async fn flush(&mut self) -> StorageResult { let buffer = self.mem_table.drain().into_parts(); let mut kv_pairs = Vec::with_capacity(buffer.len()); - for (key, key_op) in filter_with_delete_range(buffer.into_iter(), delete_ranges.iter()) { + for (key, key_op) in buffer { match key_op { // Currently, some executors do not strictly comply with these semantics. As // a workaround you may call disable the check by initializing the @@ -581,7 +578,7 @@ impl LocalStateStore for MemtableLocalState } self.inner.ingest_batch( kv_pairs, - delete_ranges, + vec![], WriteOptions { epoch: self.epoch(), table_id: self.table_id, @@ -601,14 +598,23 @@ impl LocalStateStore for MemtableLocalState async fn init(&mut self, options: InitOptions) -> StorageResult<()> { assert!( self.epoch.replace(options.epoch.curr).is_none(), - "local state store of table id {:?} is init for more than once", + "epoch in local state store of table id {:?} is init for more than once", + self.table_id + ); + assert!( + self.vnodes.replace(options.vnodes).is_none(), + "vnodes in local state store of table id {:?} is init for more than once", self.table_id ); + Ok(()) } fn seal_current_epoch(&mut self, next_epoch: u64, opts: SealCurrentEpochOptions) { assert!(!self.is_dirty()); + if let Some(value_checker) = opts.switch_op_consistency_level { + self.mem_table.op_consistency_level.update(&value_checker); + } let prev_epoch = self .epoch .replace(next_epoch) @@ -649,7 +655,7 @@ impl LocalStateStore for MemtableLocalState table_id: self.table_id, }, ) { - error!(err = ?e, "failed to write delete ranges of table watermark"); + error!(error = %e.as_report(), "failed to write delete ranges of table watermark"); } } } @@ -657,6 +663,10 @@ impl LocalStateStore for MemtableLocalState async fn try_flush(&mut self) -> StorageResult<()> { Ok(()) } + + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { + self.vnodes.replace(vnodes).unwrap() + } } #[cfg(test)] diff --git a/src/storage/src/memory.rs b/src/storage/src/memory.rs index dab25990fbc7c..4984fbf5de0a2 100644 --- a/src/storage/src/memory.rs +++ b/src/storage/src/memory.rs @@ -636,7 +636,7 @@ impl StateStore for RangeKvStateStore { fn seal_epoch(&self, _epoch: u64, _is_checkpoint: bool) {} #[allow(clippy::unused_async)] - async fn clear_shared_buffer(&self) -> StorageResult<()> { + async fn clear_shared_buffer(&self, _prev_epoch: u64) { unimplemented!("recovery not supported") } diff --git a/src/storage/src/monitor/hitmap.rs b/src/storage/src/monitor/hitmap.rs new file mode 100644 index 0000000000000..507aa4a3d2b25 --- /dev/null +++ b/src/storage/src/monitor/hitmap.rs @@ -0,0 +1,255 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; + +use risingwave_common::util::iter_util::ZipEqFast; + +#[derive(Debug, Clone)] +pub struct Hitmap { + /// For [`Hitmap`] is rarely access in multi-thread pattern, + /// the cons of false-sharing can be ignored. + data: Arc<[AtomicU64; N]>, +} + +impl Default for Hitmap { + fn default() -> Self { + let data = [(); N].map(|_| AtomicU64::default()); + let data = Arc::new(data); + Self { data } + } +} + +impl Hitmap { + pub const fn bits() -> usize { + N * u64::BITS as usize + } + + pub const fn bytes() -> usize { + N * 8 + } + + pub fn report(&self, local: &mut LocalHitmap) { + for (global, local) in self.data.iter().zip_eq_fast(local.data.iter()) { + global.fetch_or(*local, Ordering::Relaxed); + } + local.reset(); + } + + pub fn ones(&self) -> usize { + let mut res = 0; + for elem in &*self.data { + res += elem.load(Ordering::Relaxed).count_ones() as usize; + } + res + } + + pub fn zeros(&self) -> usize { + Self::bits() - self.ones() + } + + pub fn ratio(&self) -> f64 { + self.ones() as f64 / Self::bits() as f64 + } + + #[cfg(test)] + pub fn to_hex_vec(&self) -> Vec { + use itertools::Itertools; + self.data + .iter() + .map(|elem| elem.load(Ordering::Relaxed)) + .map(|v| format!("{v:016x}")) + .collect_vec() + } +} + +#[derive(Debug)] +pub struct LocalHitmap { + data: Box<[u64; N]>, +} + +impl Default for LocalHitmap { + fn default() -> Self { + Self::new() + } +} + +impl LocalHitmap { + pub const fn bits() -> usize { + N * u64::BITS as usize + } + + pub const fn bytes() -> usize { + N * 8 + } + + pub fn new() -> Self { + Self { + data: Box::new([0; N]), + } + } + + pub fn reset(&mut self) { + for elem in &mut *self.data { + *elem = 0; + } + } + + pub fn merge(&mut self, other: &mut Self) { + for (elem, e) in self.data.iter_mut().zip_eq_fast(other.data.iter()) { + *elem |= *e; + } + other.reset(); + } + + pub fn fill(&mut self, start_bit: usize, end_bit: usize) { + const MASK: usize = (1 << 6) - 1; + + let end_bit = end_bit.clamp(start_bit + 1, Self::bits()); + + let head_bits = start_bit & MASK; + let tail_bits_rev = end_bit & MASK; + + let head_elem = start_bit >> 6; + let tail_elem = end_bit >> 6; + + for i in head_elem..=std::cmp::min(tail_elem, N - 1) { + let elem = &mut self.data[i]; + let mut umask = 0u64; + if i == head_elem { + umask |= (1u64 << head_bits) - 1; + } + if i == tail_elem { + umask |= !((1u64 << tail_bits_rev) - 1); + } + *elem |= !umask; + } + } + + pub fn fill_with_range(&mut self, start: usize, end: usize, len: usize) { + let start_bit = Self::bits() * start / len; + let end_bit = Self::bits() * end / len; + self.fill(start_bit, end_bit) + } + + pub fn ones(&self) -> usize { + let mut res = 0; + for elem in &*self.data { + res += elem.count_ones() as usize; + } + res + } + + pub fn zeros(&self) -> usize { + Self::bits() - self.ones() + } + + pub fn ratio(&self) -> f64 { + self.ones() as f64 / Self::bits() as f64 + } + + #[cfg(test)] + pub fn to_hex_vec(&self) -> Vec { + use itertools::Itertools; + self.data.iter().map(|v| format!("{v:016x}")).collect_vec() + } +} + +#[cfg(debug_assertions)] +impl Drop for LocalHitmap { + fn drop(&mut self) { + if self.ones() > 0 { + panic!("LocalHitmap is not reported!"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_hitmap() { + // hex: high <== low + let g = Hitmap::<4>::default(); + + let mut h = LocalHitmap::new(); + assert_eq!( + h.to_hex_vec(), + vec![ + "0000000000000000", + "0000000000000000", + "0000000000000000", + "0000000000000000", + ] + ); + assert_eq!(h.ones(), 0); + h.fill(16, 24); + assert_eq!( + h.to_hex_vec(), + vec![ + "0000000000ff0000", + "0000000000000000", + "0000000000000000", + "0000000000000000", + ] + ); + assert_eq!(h.ones(), 8); + h.fill(32, 64); + assert_eq!( + h.to_hex_vec(), + vec![ + "ffffffff00ff0000", + "0000000000000000", + "0000000000000000", + "0000000000000000", + ] + ); + assert_eq!(h.ones(), 40); + h.fill(96, 224); + assert_eq!( + h.to_hex_vec(), + vec![ + "ffffffff00ff0000", + "ffffffff00000000", + "ffffffffffffffff", + "00000000ffffffff", + ] + ); + assert_eq!(h.ones(), 168); + h.fill(0, 256); + assert_eq!( + h.to_hex_vec(), + vec![ + "ffffffffffffffff", + "ffffffffffffffff", + "ffffffffffffffff", + "ffffffffffffffff", + ] + ); + assert_eq!(h.ones(), 256); + g.report(&mut h); + assert_eq!( + g.to_hex_vec(), + vec![ + "ffffffffffffffff", + "ffffffffffffffff", + "ffffffffffffffff", + "ffffffffffffffff", + ] + ); + assert_eq!(g.ones(), 256); + } +} diff --git a/src/storage/src/monitor/hummock_state_store_metrics.rs b/src/storage/src/monitor/hummock_state_store_metrics.rs index 45b72bd1eee86..5932185ecd5f7 100644 --- a/src/storage/src/monitor/hummock_state_store_metrics.rs +++ b/src/storage/src/monitor/hummock_state_store_metrics.rs @@ -29,6 +29,7 @@ use risingwave_common::monitor::GLOBAL_METRICS_REGISTRY; use risingwave_common::{ register_guarded_histogram_vec_with_registry, register_guarded_int_counter_vec_with_registry, }; +use thiserror_ext::AsReport; use tracing::warn; /// [`HummockStateStoreMetrics`] stores the performance and IO metrics of `XXXStore` such as @@ -76,6 +77,9 @@ pub struct HummockStateStoreMetrics { // memory pub mem_table_spill_counts: RelabeledCounterVec, + + // block statistics + pub block_efficiency_histogram: RelabeledHistogramVec, } pub static GLOBAL_HUMMOCK_STATE_STORE_METRICS: OnceLock = OnceLock::new(); @@ -287,11 +291,6 @@ impl HummockStateStoreMetrics { registry ) .unwrap(); - let spill_task_counts = RelabeledCounterVec::with_metric_level( - MetricLevel::Debug, - spill_task_counts, - metric_level, - ); let spill_task_size = register_int_counter_vec_with_registry!( "state_store_spill_task_size", @@ -300,11 +299,6 @@ impl HummockStateStoreMetrics { registry ) .unwrap(); - let spill_task_size = RelabeledCounterVec::with_metric_level( - MetricLevel::Debug, - spill_task_size, - metric_level, - ); let uploader_uploading_task_size = GenericGauge::new( "state_store_uploader_uploading_task_size", @@ -323,10 +317,11 @@ impl HummockStateStoreMetrics { ) .unwrap(); let read_req_bloom_filter_positive_counts = - RelabeledGuardedIntCounterVec::with_metric_level( + RelabeledGuardedIntCounterVec::with_metric_level_relabel_n( MetricLevel::Info, read_req_bloom_filter_positive_counts, metric_level, + 1, ); let read_req_positive_but_non_exist_counts = register_guarded_int_counter_vec_with_registry!( @@ -371,6 +366,19 @@ impl HummockStateStoreMetrics { metric_level, ); + let opts = histogram_opts!( + "block_efficiency_histogram", + "Access ratio of in-memory block.", + exponential_buckets(0.001, 2.0, 11).unwrap(), + ); + let block_efficiency_histogram = + register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap(); + let block_efficiency_histogram = RelabeledHistogramVec::with_metric_level( + MetricLevel::Info, + block_efficiency_histogram, + metric_level, + ); + Self { bloom_filter_true_negative_counts, bloom_filter_check_counts, @@ -396,6 +404,8 @@ impl HummockStateStoreMetrics { spill_task_size_from_unsealed: spill_task_size.with_label_values(&["unsealed"]), uploader_uploading_task_size, mem_table_spill_counts, + + block_efficiency_histogram, } } @@ -516,8 +526,8 @@ pub fn monitor_cache(memory_collector: Arc) { let collector = Box::new(StateStoreCollector::new(memory_collector)); if let Err(e) = GLOBAL_METRICS_REGISTRY.register(collector) { warn!( - "unable to monitor cache. May have been registered if in all-in-one deployment: {:?}", - e + "unable to monitor cache. May have been registered if in all-in-one deployment: {}", + e.as_report() ); } } diff --git a/src/storage/src/monitor/mod.rs b/src/storage/src/monitor/mod.rs index 053cc72cf8130..1849f272d8473 100644 --- a/src/storage/src/monitor/mod.rs +++ b/src/storage/src/monitor/mod.rs @@ -28,6 +28,9 @@ pub use compactor_metrics::*; mod local_metrics; pub use local_metrics::*; + +mod hitmap; +pub use hitmap::*; pub use risingwave_object_store::object::object_metrics::{ ObjectStoreMetrics, GLOBAL_OBJECT_STORE_METRICS, }; diff --git a/src/storage/src/monitor/monitored_storage_metrics.rs b/src/storage/src/monitor/monitored_storage_metrics.rs index b2cf2186deab8..009cd7308df9e 100644 --- a/src/storage/src/monitor/monitored_storage_metrics.rs +++ b/src/storage/src/monitor/monitored_storage_metrics.rs @@ -127,7 +127,7 @@ impl MonitoredStorageMetrics { let opts = histogram_opts!( "state_store_iter_item", "Total bytes gotten from state store scan(), for calculating read throughput", - size_buckets + size_buckets.clone(), ); let iter_item = register_histogram_vec_with_registry!(opts, &["table_id"], registry).unwrap(); @@ -189,14 +189,14 @@ impl MonitoredStorageMetrics { let opts = histogram_opts!( "state_store_sync_duration", "Histogram of time spent on compacting shared buffer to remote storage", - time_buckets.clone() + time_buckets, ); let sync_duration = register_histogram_with_registry!(opts, registry).unwrap(); let opts = histogram_opts!( "state_store_sync_size", "Total size of upload to l0 every epoch", - time_buckets + size_buckets, ); let sync_size = register_histogram_with_registry!(opts, registry).unwrap(); diff --git a/src/storage/src/monitor/monitored_store.rs b/src/storage/src/monitor/monitored_store.rs index facea1f090903..e1110df630595 100644 --- a/src/storage/src/monitor/monitored_store.rs +++ b/src/storage/src/monitor/monitored_store.rs @@ -12,16 +12,17 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::ops::Bound; use std::sync::Arc; use await_tree::InstrumentAwait; use bytes::Bytes; use futures::{Future, TryFutureExt, TryStreamExt}; use futures_async_stream::try_stream; +use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::TableId; use risingwave_hummock_sdk::key::{TableKey, TableKeyRange}; use risingwave_hummock_sdk::HummockReadEpoch; +use thiserror_ext::AsReport; use tokio::time::Instant; use tracing::error; @@ -99,7 +100,7 @@ impl MonitoredStateStore { let iter_stream = iter_stream_future .verbose_instrument_await("store_create_iter") .await - .inspect_err(|e| error!("Failed in iter: {:?}", e))?; + .inspect_err(|e| error!(error = %e.as_report(), "Failed in iter"))?; self.storage_metrics .iter_init_duration @@ -153,7 +154,7 @@ impl MonitoredStateStore { .verbose_instrument_await("store_get") .instrument(tracing::trace_span!("store_get")) .await - .inspect_err(|e| error!("Failed in get: {:?}", e))?; + .inspect_err(|e| error!(error = %e.as_report(), "Failed in get"))?; timer.observe_duration(); @@ -260,13 +261,8 @@ impl LocalStateStore for MonitoredStateStore { self.inner.delete(key, old_val) } - fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> impl Future> + Send + '_ { - self.inner - .flush(delete_ranges) - .verbose_instrument_await("store_flush") + fn flush(&mut self) -> impl Future> + Send + '_ { + self.inner.flush().verbose_instrument_await("store_flush") } fn epoch(&self) -> u64 { @@ -291,6 +287,10 @@ impl LocalStateStore for MonitoredStateStore { .try_flush() .verbose_instrument_await("store_try_flush") } + + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { + self.inner.update_vnode_bitmap(vnodes) + } } impl StateStore for MonitoredStateStore { @@ -303,7 +303,7 @@ impl StateStore for MonitoredStateStore { self.inner .try_wait_epoch(epoch) .verbose_instrument_await("store_wait_epoch") - .inspect_err(|e| error!("Failed in wait_epoch: {:?}", e)) + .inspect_err(|e| error!(error = %e.as_report(), "Failed in wait_epoch")) } async fn sync(&self, epoch: u64) -> StorageResult { @@ -315,7 +315,7 @@ impl StateStore for MonitoredStateStore { .sync(epoch) .instrument_await("store_sync") .await - .inspect_err(|e| error!("Failed in sync: {:?}", e))?; + .inspect_err(|e| error!(error = %e.as_report(), "Failed in sync"))?; timer.observe_duration(); if sync_result.sync_size != 0 { self.storage_metrics @@ -336,11 +336,10 @@ impl StateStore for MonitoredStateStore { panic!("the state store is already monitored") } - fn clear_shared_buffer(&self) -> impl Future> + Send + '_ { + fn clear_shared_buffer(&self, prev_epoch: u64) -> impl Future + Send + '_ { self.inner - .clear_shared_buffer() + .clear_shared_buffer(prev_epoch) .verbose_instrument_await("store_clear_shared_buffer") - .inspect_err(|e| error!("Failed in clear_shared_buffer: {:?}", e)) } async fn new_local(&self, option: NewLocalOptions) -> Self::Local { @@ -393,7 +392,7 @@ impl MonitoredStateStoreIter { while let Some((key, value)) = inner .try_next() .await - .inspect_err(|e| error!("Failed in next: {:?}", e))? + .inspect_err(|e| error!(error = %e.as_report(), "Failed in next"))? { stats.total_items += 1; stats.total_size += key.encoded_len() + value.len(); diff --git a/src/storage/src/monitor/traced_store.rs b/src/storage/src/monitor/traced_store.rs index 7821572076bbe..de55143dd7d73 100644 --- a/src/storage/src/monitor/traced_store.rs +++ b/src/storage/src/monitor/traced_store.rs @@ -11,18 +11,19 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - -use std::ops::Bound; +use std::sync::Arc; use bytes::Bytes; use futures::{Future, TryFutureExt, TryStreamExt}; use futures_async_stream::try_stream; +use risingwave_common::buffer::Bitmap; use risingwave_hummock_sdk::key::{TableKey, TableKeyRange}; use risingwave_hummock_sdk::HummockReadEpoch; use risingwave_hummock_trace::{ init_collector, should_use_trace, ConcurrentId, MayTraceSpan, OperationResult, StorageType, TraceResult, TraceSpan, TracedBytes, TracedSealCurrentEpochOptions, LOCAL_ID, }; +use thiserror_ext::AsReport; use super::identity; use crate::error::{StorageError, StorageResult}; @@ -176,12 +177,9 @@ impl LocalStateStore for TracedStateStore { res } - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> StorageResult { - let span = TraceSpan::new_flush_span(delete_ranges.clone(), self.storage_type); - let res = self.inner.flush(delete_ranges).await; + async fn flush(&mut self) -> StorageResult { + let span = TraceSpan::new_flush_span(self.storage_type); + let res = self.inner.flush().await; span.may_send_result(OperationResult::Flush( res.as_ref().map(|o: &usize| *o).into(), )); @@ -223,6 +221,11 @@ impl LocalStateStore for TracedStateStore { span.may_send_result(OperationResult::TryFlush(res.as_ref().map(|o| *o).into())); res } + + // TODO: add trace span + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { + self.inner.update_vnode_bitmap(vnodes) + } } impl StateStore for TracedStateStore { @@ -254,13 +257,9 @@ impl StateStore for TracedStateStore { self.inner.seal_epoch(epoch, is_checkpoint); } - async fn clear_shared_buffer(&self) -> StorageResult<()> { - let span = TraceSpan::new_clear_shared_buffer_span(); - let res = self.inner.clear_shared_buffer().await; - span.may_send_result(OperationResult::ClearSharedBuffer( - res.as_ref().map(|o| *o).into(), - )); - res + async fn clear_shared_buffer(&self, prev_epoch: u64) { + let _span = TraceSpan::new_clear_shared_buffer_span(prev_epoch); + self.inner.clear_shared_buffer(prev_epoch).await; } async fn new_local(&self, options: NewLocalOptions) -> Self::Local { @@ -357,7 +356,7 @@ impl TracedStateStoreIter { while let Some((key, value)) = inner .try_next() .await - .inspect_err(|e| tracing::error!("Failed in next: {:?}", e))? + .inspect_err(|e| tracing::error!(error = %e.as_report(), "Failed in next"))? { self.span.may_send_iter_next(); self.span diff --git a/src/storage/src/opts.rs b/src/storage/src/opts.rs index 349b25c7876d3..60a787f46c481 100644 --- a/src/storage/src/opts.rs +++ b/src/storage/src/opts.rs @@ -15,7 +15,7 @@ use risingwave_common::config::{ extract_storage_memory_config, ObjectStoreConfig, RwConfig, StorageMemoryConfig, }; -use risingwave_common::system_param::reader::SystemParamsReader; +use risingwave_common::system_param::reader::{SystemParamsRead, SystemParamsReader}; use risingwave_common::system_param::system_params_for_test; #[derive(Clone, Debug)] @@ -130,7 +130,7 @@ pub struct StorageOpts { pub compactor_max_sst_size: u64, /// enable FastCompactorRunner. pub enable_fast_compaction: bool, - pub check_fast_compaction_result: bool, + pub check_compaction_result: bool, pub max_preload_io_retry_times: usize, pub compactor_fast_max_compact_delete_ratio: u32, pub compactor_fast_max_compact_task_size: u64, @@ -253,7 +253,7 @@ impl From<(&RwConfig, &SystemParamsReader, &StorageMemoryConfig)> for StorageOpt compactor_max_task_multiplier: c.storage.compactor_max_task_multiplier, compactor_max_sst_size: c.storage.compactor_max_sst_size, enable_fast_compaction: c.storage.enable_fast_compaction, - check_fast_compaction_result: c.storage.check_fast_compaction_result, + check_compaction_result: c.storage.check_compaction_result, mem_table_spill_threshold: c.storage.mem_table_spill_threshold, object_store_config: c.storage.object_store.clone(), compactor_fast_max_compact_delete_ratio: c diff --git a/src/storage/src/panic_store.rs b/src/storage/src/panic_store.rs index f26d5edeb37de..7e5985bb6aefe 100644 --- a/src/storage/src/panic_store.rs +++ b/src/storage/src/panic_store.rs @@ -14,10 +14,12 @@ use std::ops::Bound; use std::pin::Pin; +use std::sync::Arc; use std::task::{Context, Poll}; use bytes::Bytes; use futures::Stream; +use risingwave_common::buffer::Bitmap; use risingwave_hummock_sdk::key::{TableKey, TableKeyRange}; use risingwave_hummock_sdk::HummockReadEpoch; @@ -109,10 +111,7 @@ impl LocalStateStore for PanicStateStore { } #[allow(clippy::unused_async)] - async fn flush( - &mut self, - _delete_ranges: Vec<(Bound, Bound)>, - ) -> StorageResult { + async fn flush(&mut self) -> StorageResult { panic!("should not operate on the panic state store!"); } @@ -137,6 +136,10 @@ impl LocalStateStore for PanicStateStore { async fn try_flush(&mut self) -> StorageResult<()> { panic!("should not operate on the panic state store!"); } + + fn update_vnode_bitmap(&mut self, _vnodes: Arc) -> Arc { + panic!("should not operate on the panic state store!"); + } } impl StateStore for PanicStateStore { @@ -157,7 +160,7 @@ impl StateStore for PanicStateStore { } #[allow(clippy::unused_async)] - async fn clear_shared_buffer(&self) -> StorageResult<()> { + async fn clear_shared_buffer(&self, _prev_epoch: u64) { panic!("should not clear shared buffer from the panic state store!"); } diff --git a/src/storage/src/row_serde/mod.rs b/src/storage/src/row_serde/mod.rs index 96ef7be735244..138d2361468c5 100644 --- a/src/storage/src/row_serde/mod.rs +++ b/src/storage/src/row_serde/mod.rs @@ -30,14 +30,18 @@ pub fn find_columns_by_ids( table_columns: &[ColumnDesc], column_ids: &[ColumnId], ) -> (Vec, Vec) { - let mut table_columns = table_columns + if column_ids.is_empty() { + // shortcut + return (vec![], vec![]); + } + let id_to_columns = table_columns .iter() .enumerate() .map(|(index, c)| (c.column_id, (c.clone(), index))) .collect::>(); column_ids .iter() - .map(|id| table_columns.remove(id).unwrap()) + .map(|id| id_to_columns.get(id).expect("column id not found").clone()) .unzip() } @@ -95,7 +99,9 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_column_type: Normal, + additional_column: AdditionalColumn { + column_type: None, + }, version: Pr13707, }, ColumnDesc { @@ -106,7 +112,9 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_column_type: Normal, + additional_column: AdditionalColumn { + column_type: None, + }, version: Pr13707, }, ], @@ -137,7 +145,9 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_column_type: Normal, + additional_column: AdditionalColumn { + column_type: None, + }, version: Pr13707, }, ColumnDesc { @@ -148,7 +158,9 @@ mod test { type_name: "", generated_or_default_column: None, description: None, - additional_column_type: Normal, + additional_column: AdditionalColumn { + column_type: None, + }, version: Pr13707, }, ], diff --git a/src/storage/src/storage_failpoints/test_iterator.rs b/src/storage/src/storage_failpoints/test_iterator.rs index 9eebdeaa64d6d..7b1aa31c808cd 100644 --- a/src/storage/src/storage_failpoints/test_iterator.rs +++ b/src/storage/src/storage_failpoints/test_iterator.rs @@ -24,8 +24,8 @@ use crate::hummock::iterator::test_utils::{ TEST_KEYS_COUNT, }; use crate::hummock::iterator::{ - BackwardConcatIterator, BackwardUserIterator, ConcatIterator, HummockIterator, - UnorderedMergeIteratorInner, UserIterator, + BackwardConcatIterator, BackwardUserIterator, ConcatIterator, HummockIterator, MergeIterator, + UserIterator, }; use crate::hummock::sstable::SstableIteratorReadOptions; use crate::hummock::test_utils::{ @@ -175,7 +175,7 @@ async fn test_failpoints_merge_invalid_key() { ) .await; let tables = vec![table0, table1]; - let mut mi = UnorderedMergeIteratorInner::new({ + let mut mi = MergeIterator::new({ let mut iters = vec![]; for table in tables { iters.push(SstableIterator::new( @@ -223,7 +223,7 @@ async fn test_failpoints_backward_merge_invalid_key() { ) .await; let tables = vec![table0, table1]; - let mut mi = UnorderedMergeIteratorInner::new({ + let mut mi = MergeIterator::new({ let mut iters = vec![]; for table in tables { iters.push(BackwardSstableIterator::new(table, sstable_store.clone())); @@ -279,7 +279,7 @@ async fn test_failpoints_user_read_err() { ), ]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut ui = UserIterator::for_test(mi, (Unbounded, Unbounded)); ui.rewind().await.unwrap(); @@ -331,7 +331,7 @@ async fn test_failpoints_backward_user_read_err() { BackwardSstableIterator::new(table1, sstable_store.clone()), ]; - let mi = UnorderedMergeIteratorInner::new(iters); + let mi = MergeIterator::new(iters); let mut ui = BackwardUserIterator::for_test(mi, (Unbounded, Unbounded)); ui.rewind().await.unwrap(); @@ -400,7 +400,7 @@ async fn test_failpoints_compactor_iterator_recreate() { let table = sstable_store.sstable(&info, &mut stats).await.unwrap(); let mut sstable_iter = SstableStreamIterator::new( - table.value().meta.block_metas.clone(), + table.meta.block_metas.clone(), info, HashSet::from_iter(std::iter::once(0)), 0, diff --git a/src/storage/src/store.rs b/src/storage/src/store.rs index 2d1bc73fffb47..e5b096de47196 100644 --- a/src/storage/src/store.rs +++ b/src/storage/src/store.rs @@ -14,6 +14,7 @@ use std::collections::HashMap; use std::default::Default; +use std::fmt::{Debug, Formatter}; use std::future::Future; use std::ops::Bound; use std::sync::{Arc, LazyLock}; @@ -22,6 +23,7 @@ use bytes::Bytes; use futures::{Stream, StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use prost::Message; +use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::{TableId, TableOption}; use risingwave_common::util::epoch::{Epoch, EpochPair}; use risingwave_hummock_sdk::key::{FullKey, TableKey, TableKeyRange}; @@ -30,8 +32,8 @@ use risingwave_hummock_sdk::table_watermark::{ }; use risingwave_hummock_sdk::{HummockReadEpoch, LocalSstableInfo}; use risingwave_hummock_trace::{ - TracedInitOptions, TracedNewLocalOptions, TracedOpConsistencyLevel, TracedPrefetchOptions, - TracedReadOptions, TracedSealCurrentEpochOptions, TracedWriteOptions, + TracedBitmap, TracedInitOptions, TracedNewLocalOptions, TracedOpConsistencyLevel, + TracedPrefetchOptions, TracedReadOptions, TracedSealCurrentEpochOptions, TracedWriteOptions, }; use crate::error::{StorageError, StorageResult}; @@ -190,7 +192,7 @@ pub trait StateStore: StateStoreRead + StaticSendSync + Clone { /// Clears contents in shared buffer. /// This method should only be called when dropping all actors in the local compute node. - fn clear_shared_buffer(&self) -> impl Future> + Send + '_; + fn clear_shared_buffer(&self, prev_epoch: u64) -> impl Future + Send + '_; fn new_local(&self, option: NewLocalOptions) -> impl Future + Send + '_; @@ -235,10 +237,7 @@ pub trait LocalStateStore: StaticSendSync { /// than the given `epoch` will be deleted. fn delete(&mut self, key: TableKey, old_val: Bytes) -> StorageResult<()>; - fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> impl Future> + Send + '_; + fn flush(&mut self) -> impl Future> + Send + '_; fn try_flush(&mut self) -> impl Future> + Send + '_; fn epoch(&self) -> u64; @@ -272,6 +271,10 @@ pub trait LocalStateStore: StaticSendSync { key_range: TableKeyRange, read_options: ReadOptions, ) -> impl Future> + Send + '_; + + // Updates the vnode bitmap corresponding to the local state store + // Returns the previous vnode bitmap + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc; } /// If `prefetch` is true, prefetch will be enabled. Prefetching may increase the memory @@ -409,6 +412,41 @@ pub enum OpConsistencyLevel { ConsistentOldValue(Arc), } +impl Debug for OpConsistencyLevel { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + OpConsistencyLevel::Inconsistent => f.write_str("OpConsistencyLevel::Inconsistent"), + OpConsistencyLevel::ConsistentOldValue(_) => { + f.write_str("OpConsistencyLevel::ConsistentOldValue") + } + } + } +} + +impl PartialEq for OpConsistencyLevel { + fn eq(&self, other: &Self) -> bool { + matches!( + (self, other), + ( + OpConsistencyLevel::Inconsistent, + OpConsistencyLevel::Inconsistent + ) | ( + OpConsistencyLevel::ConsistentOldValue(_), + OpConsistencyLevel::ConsistentOldValue(_), + ) + ) + } +} + +impl Eq for OpConsistencyLevel {} + +impl OpConsistencyLevel { + pub fn update(&mut self, new_level: &OpConsistencyLevel) { + assert_ne!(self, new_level); + *self = new_level.clone() + } +} + #[derive(Clone, Default)] pub struct NewLocalOptions { pub table_id: TableId, @@ -501,17 +539,14 @@ impl NewLocalOptions { #[derive(Clone)] pub struct InitOptions { pub epoch: EpochPair, -} -impl InitOptions { - pub fn new_with_epoch(epoch: EpochPair) -> Self { - Self { epoch } - } + /// The vnode bitmap for the local state store instance + pub vnodes: Arc, } -impl From for InitOptions { - fn from(value: EpochPair) -> Self { - Self { epoch: value } +impl InitOptions { + pub fn new(epoch: EpochPair, vnodes: Arc) -> Self { + Self { epoch, vnodes } } } @@ -519,6 +554,7 @@ impl From for TracedInitOptions { fn from(value: InitOptions) -> Self { TracedInitOptions { epoch: value.epoch.into(), + vnodes: TracedBitmap::from(value.vnodes.as_ref().clone()), } } } @@ -527,6 +563,7 @@ impl From for InitOptions { fn from(value: TracedInitOptions) -> Self { InitOptions { epoch: value.epoch.into(), + vnodes: Arc::new(Bitmap::from(value.vnodes)), } } } @@ -534,6 +571,7 @@ impl From for InitOptions { #[derive(Clone, Debug)] pub struct SealCurrentEpochOptions { pub table_watermarks: Option<(WatermarkDirection, Vec)>, + pub switch_op_consistency_level: Option, } impl From for TracedSealCurrentEpochOptions { @@ -548,6 +586,9 @@ impl From for TracedSealCurrentEpochOptions { .collect(), ) }), + switch_op_consistency_level: value + .switch_op_consistency_level + .map(|level| matches!(level, OpConsistencyLevel::ConsistentOldValue(_))), } } } @@ -572,24 +613,22 @@ impl From for SealCurrentEpochOptions { .collect(), ) }), + switch_op_consistency_level: value.switch_op_consistency_level.map(|enable| { + if enable { + OpConsistencyLevel::ConsistentOldValue(CHECK_BYTES_EQUAL.clone()) + } else { + OpConsistencyLevel::Inconsistent + } + }), } } } impl SealCurrentEpochOptions { - pub fn new(watermarks: Vec, direction: WatermarkDirection) -> Self { - Self { - table_watermarks: Some((direction, watermarks)), - } - } - - pub fn no_watermark() -> Self { + pub fn for_test() -> Self { Self { table_watermarks: None, + switch_op_consistency_level: None, } } - - pub fn for_test() -> Self { - Self::no_watermark() - } } diff --git a/src/storage/src/store_impl.rs b/src/storage/src/store_impl.rs index 86adbe13b01bd..50fe81d53ed54 100644 --- a/src/storage/src/store_impl.rs +++ b/src/storage/src/store_impl.rs @@ -28,7 +28,7 @@ use crate::hummock::file_cache::preclude::*; use crate::hummock::hummock_meta_client::MonitoredHummockMetaClient; use crate::hummock::{ set_foyer_metrics_registry, FileCache, FileCacheConfig, HummockError, HummockStorage, - RecentFilter, SstableStore, + RecentFilter, SstableStore, SstableStoreConfig, }; use crate::memory::sled::SledStateStore; use crate::memory::MemoryStateStore; @@ -203,10 +203,12 @@ pub mod verify { use std::fmt::Debug; use std::future::Future; use std::ops::{Bound, Deref}; + use std::sync::Arc; use bytes::Bytes; use futures::{pin_mut, TryStreamExt}; use futures_async_stream::try_stream; + use risingwave_common::buffer::Bitmap; use risingwave_hummock_sdk::key::{TableKey, TableKeyRange}; use risingwave_hummock_sdk::HummockReadEpoch; use tracing::log::warn; @@ -415,14 +417,11 @@ pub mod verify { Ok(()) } - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> StorageResult { + async fn flush(&mut self) -> StorageResult { if let Some(expected) = &mut self.expected { - expected.flush(delete_ranges.clone()).await?; + expected.flush().await?; } - self.actual.flush(delete_ranges).await + self.actual.flush().await } async fn try_flush(&mut self) -> StorageResult<()> { @@ -462,6 +461,14 @@ pub mod verify { } ret } + + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { + let ret = self.actual.update_vnode_bitmap(vnodes.clone()); + if let Some(expected) = &mut self.expected { + assert_eq!(ret, expected.update_vnode_bitmap(vnodes)); + } + ret + } } impl StateStore for VerifyStateStore { @@ -485,8 +492,8 @@ pub mod verify { self.actual.seal_epoch(epoch, is_checkpoint) } - fn clear_shared_buffer(&self) -> impl Future> + Send + '_ { - self.actual.clear_shared_buffer() + fn clear_shared_buffer(&self, prev_epoch: u64) -> impl Future + Send + '_ { + self.actual.clear_shared_buffer(prev_epoch) } async fn new_local(&self, option: NewLocalOptions) -> Self::Local { @@ -610,18 +617,19 @@ impl StateStoreImpl { ) .await; - let sstable_store = Arc::new(SstableStore::new( - Arc::new(object_store), - opts.data_directory.to_string(), - opts.block_cache_capacity_mb * (1 << 20), - opts.meta_cache_capacity_mb * (1 << 20), - opts.high_priority_ratio, - opts.prefetch_buffer_capacity_mb * (1 << 20), - opts.max_prefetch_block_number, + let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig { + store: Arc::new(object_store), + path: opts.data_directory.to_string(), + block_cache_capacity: opts.block_cache_capacity_mb * (1 << 20), + meta_cache_capacity: opts.meta_cache_capacity_mb * (1 << 20), + high_priority_ratio: opts.high_priority_ratio, + prefetch_buffer_capacity: opts.prefetch_buffer_capacity_mb * (1 << 20), + max_prefetch_block_number: opts.max_prefetch_block_number, data_file_cache, meta_file_cache, recent_filter, - )); + state_store_metrics: state_store_metrics.clone(), + })); let notification_client = RpcNotificationClient::new(hummock_meta_client.get_inner().clone()); let key_filter_manager = Arc::new(RpcFilterKeyExtractorManager::new(Box::new( @@ -684,12 +692,14 @@ impl AsHummock for SledStateStore { #[cfg(debug_assertions)] pub mod boxed_state_store { use std::future::Future; - use std::ops::{Bound, Deref, DerefMut}; + use std::ops::{Deref, DerefMut}; + use std::sync::Arc; use bytes::Bytes; use dyn_clone::{clone_trait_object, DynClone}; use futures::stream::BoxStream; use futures::StreamExt; + use risingwave_common::buffer::Bitmap; use risingwave_hummock_sdk::key::{TableKey, TableKeyRange}; use risingwave_hummock_sdk::HummockReadEpoch; @@ -772,10 +782,7 @@ pub mod boxed_state_store { fn delete(&mut self, key: TableKey, old_val: Bytes) -> StorageResult<()>; - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> StorageResult; + async fn flush(&mut self) -> StorageResult; async fn try_flush(&mut self) -> StorageResult<()>; @@ -786,6 +793,8 @@ pub mod boxed_state_store { async fn init(&mut self, epoch: InitOptions) -> StorageResult<()>; fn seal_current_epoch(&mut self, next_epoch: u64, opts: SealCurrentEpochOptions); + + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc; } #[async_trait::async_trait] @@ -827,11 +836,8 @@ pub mod boxed_state_store { self.delete(key, old_val) } - async fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> StorageResult { - self.flush(delete_ranges).await + async fn flush(&mut self) -> StorageResult { + self.flush().await } async fn try_flush(&mut self) -> StorageResult<()> { @@ -853,6 +859,10 @@ pub mod boxed_state_store { fn seal_current_epoch(&mut self, next_epoch: u64, opts: SealCurrentEpochOptions) { self.seal_current_epoch(next_epoch, opts) } + + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { + self.update_vnode_bitmap(vnodes) + } } pub type BoxDynamicDispatchedLocalStateStore = Box; @@ -897,11 +907,8 @@ pub mod boxed_state_store { self.deref_mut().delete(key, old_val) } - fn flush( - &mut self, - delete_ranges: Vec<(Bound, Bound)>, - ) -> impl Future> + Send + '_ { - self.deref_mut().flush(delete_ranges) + fn flush(&mut self) -> impl Future> + Send + '_ { + self.deref_mut().flush() } fn try_flush(&mut self) -> impl Future> + Send + '_ { @@ -926,6 +933,10 @@ pub mod boxed_state_store { fn seal_current_epoch(&mut self, next_epoch: u64, opts: SealCurrentEpochOptions) { self.deref_mut().seal_current_epoch(next_epoch, opts) } + + fn update_vnode_bitmap(&mut self, vnodes: Arc) -> Arc { + self.deref_mut().update_vnode_bitmap(vnodes) + } } // For global StateStore @@ -938,7 +949,7 @@ pub mod boxed_state_store { fn seal_epoch(&self, epoch: u64, is_checkpoint: bool); - async fn clear_shared_buffer(&self) -> StorageResult<()>; + async fn clear_shared_buffer(&self, prev_epoch: u64); async fn new_local(&self, option: NewLocalOptions) -> BoxDynamicDispatchedLocalStateStore; @@ -959,8 +970,8 @@ pub mod boxed_state_store { self.seal_epoch(epoch, is_checkpoint); } - async fn clear_shared_buffer(&self) -> StorageResult<()> { - self.clear_shared_buffer().await + async fn clear_shared_buffer(&self, prev_epoch: u64) { + self.clear_shared_buffer(prev_epoch).await } async fn new_local(&self, option: NewLocalOptions) -> BoxDynamicDispatchedLocalStateStore { @@ -1029,8 +1040,8 @@ pub mod boxed_state_store { self.deref().sync(epoch) } - fn clear_shared_buffer(&self) -> impl Future> + Send + '_ { - self.deref().clear_shared_buffer() + fn clear_shared_buffer(&self, prev_epoch: u64) -> impl Future + Send + '_ { + self.deref().clear_shared_buffer(prev_epoch) } fn seal_epoch(&self, epoch: u64, is_checkpoint: bool) { diff --git a/src/storage/src/table/batch_table/storage_table.rs b/src/storage/src/table/batch_table/storage_table.rs index 0c55bf79ffa4b..031dcbbf1c42b 100644 --- a/src/storage/src/table/batch_table/storage_table.rs +++ b/src/storage/src/table/batch_table/storage_table.rs @@ -146,11 +146,7 @@ impl StorageTableInner { .collect_vec(); let table_option = TableOption { - retention_seconds: if table_desc.retention_seconds > 0 { - Some(table_desc.retention_seconds) - } else { - None - }, + retention_seconds: table_desc.retention_seconds, }; let value_indices = table_desc .get_value_indices() @@ -159,14 +155,7 @@ impl StorageTableInner { .collect_vec(); let prefix_hint_len = table_desc.get_read_prefix_len_hint() as usize; let versioned = table_desc.versioned; - let dist_key_in_pk_indices = table_desc - .dist_key_in_pk_indices - .iter() - .map(|&k| k as usize) - .collect_vec(); - let vnode_col_idx_in_pk = table_desc.vnode_col_idx_in_pk.map(|k| k as usize); - let distribution = - TableDistribution::new(vnodes, dist_key_in_pk_indices, vnode_col_idx_in_pk); + let distribution = TableDistribution::new_from_storage_table_desc(vnodes, table_desc); Self::new_inner( store, diff --git a/src/storage/src/table/mod.rs b/src/storage/src/table/mod.rs index 105a2f1f18604..a4d9f958b8246 100644 --- a/src/storage/src/table/mod.rs +++ b/src/storage/src/table/mod.rs @@ -15,186 +15,21 @@ pub mod batch_table; pub mod merge_sort; -use std::mem::replace; use std::ops::Deref; -use std::sync::{Arc, LazyLock}; use bytes::Bytes; use futures::{Stream, StreamExt}; -use itertools::Itertools; -use risingwave_common::array::{Array, DataChunk, PrimitiveArray}; -use risingwave_common::buffer::{Bitmap, BitmapBuilder}; +use risingwave_common::array::DataChunk; use risingwave_common::catalog::Schema; +pub use risingwave_common::hash::table_distribution::*; use risingwave_common::hash::VirtualNode; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_hummock_sdk::key::TableKey; -use tracing::warn; use crate::error::StorageResult; -/// For tables without distribution (singleton), the `DEFAULT_VNODE` is encoded. -pub const DEFAULT_VNODE: VirtualNode = VirtualNode::ZERO; - -#[derive(Debug, Clone)] -enum ComputeVnode { - Singleton, - DistKeyIndices { - /// Indices of distribution key for computing vnode, based on the all columns of the table. - dist_key_in_pk_indices: Vec, - }, - VnodeColumnIndex { - /// Indices of vnode columns. - vnode_col_idx_in_pk: usize, - }, -} - -#[derive(Debug, Clone)] -/// Represents the distribution for a specific table instance. -pub struct TableDistribution { - /// The way to compute vnode provided primary key - compute_vnode: ComputeVnode, - - /// Virtual nodes that the table is partitioned into. - vnodes: Arc, -} - -pub const SINGLETON_VNODE: VirtualNode = DEFAULT_VNODE; - -impl TableDistribution { - pub fn new( - vnodes: Option>, - dist_key_in_pk_indices: Vec, - vnode_col_idx_in_pk: Option, - ) -> Self { - let compute_vnode = if let Some(vnode_col_idx_in_pk) = vnode_col_idx_in_pk { - ComputeVnode::VnodeColumnIndex { - vnode_col_idx_in_pk, - } - } else if !dist_key_in_pk_indices.is_empty() { - ComputeVnode::DistKeyIndices { - dist_key_in_pk_indices, - } - } else { - ComputeVnode::Singleton - }; - - let vnodes = vnodes.unwrap_or_else(Self::singleton_vnode_bitmap); - if let ComputeVnode::Singleton = &compute_vnode { - if &vnodes != Self::singleton_vnode_bitmap_ref() { - warn!( - ?vnodes, - "singleton distribution get non-singleton vnode bitmap" - ); - } - } - - Self { - compute_vnode, - vnodes, - } - } - - pub fn is_singleton(&self) -> bool { - matches!(&self.compute_vnode, ComputeVnode::Singleton) - } - - pub fn singleton_vnode_bitmap_ref() -> &'static Arc { - /// A bitmap that only the default vnode is set. - static SINGLETON_VNODES: LazyLock> = LazyLock::new(|| { - let mut vnodes = BitmapBuilder::zeroed(VirtualNode::COUNT); - vnodes.set(SINGLETON_VNODE.to_index(), true); - vnodes.finish().into() - }); - - SINGLETON_VNODES.deref() - } - - pub fn singleton_vnode_bitmap() -> Arc { - Self::singleton_vnode_bitmap_ref().clone() - } - - pub fn all_vnodes_ref() -> &'static Arc { - /// A bitmap that all vnodes are set. - static ALL_VNODES: LazyLock> = - LazyLock::new(|| Bitmap::ones(VirtualNode::COUNT).into()); - &ALL_VNODES - } - - pub fn all_vnodes() -> Arc { - Self::all_vnodes_ref().clone() - } - - /// Distribution that accesses all vnodes, mainly used for tests. - pub fn all(dist_key_in_pk_indices: Vec) -> Self { - Self { - compute_vnode: ComputeVnode::DistKeyIndices { - dist_key_in_pk_indices, - }, - vnodes: Self::all_vnodes(), - } - } - - /// Fallback distribution for singleton or tests. - pub fn singleton() -> Self { - Self { - compute_vnode: ComputeVnode::Singleton, - vnodes: Self::singleton_vnode_bitmap(), - } - } - - pub fn update_vnode_bitmap(&mut self, new_vnodes: Arc) -> Arc { - if self.is_singleton() && &new_vnodes != Self::singleton_vnode_bitmap_ref() { - warn!(?new_vnodes, "update vnode on singleton distribution"); - } - assert_eq!(self.vnodes.len(), new_vnodes.len()); - replace(&mut self.vnodes, new_vnodes) - } - - pub fn vnodes(&self) -> &Arc { - &self.vnodes - } - - /// Get vnode value with given primary key. - pub fn compute_vnode_by_pk(&self, pk: impl Row) -> VirtualNode { - match &self.compute_vnode { - ComputeVnode::Singleton => SINGLETON_VNODE, - ComputeVnode::DistKeyIndices { - dist_key_in_pk_indices, - } => compute_vnode(pk, dist_key_in_pk_indices, &self.vnodes), - ComputeVnode::VnodeColumnIndex { - vnode_col_idx_in_pk, - } => get_vnode_from_row(pk, *vnode_col_idx_in_pk, &self.vnodes), - } - } - - pub fn try_compute_vnode_by_pk_prefix(&self, pk_prefix: impl Row) -> Option { - match &self.compute_vnode { - ComputeVnode::Singleton => Some(SINGLETON_VNODE), - ComputeVnode::DistKeyIndices { - dist_key_in_pk_indices, - } => dist_key_in_pk_indices - .iter() - .all(|&d| d < pk_prefix.len()) - .then(|| compute_vnode(pk_prefix, dist_key_in_pk_indices, &self.vnodes)), - ComputeVnode::VnodeColumnIndex { - vnode_col_idx_in_pk, - } => { - if *vnode_col_idx_in_pk >= pk_prefix.len() { - None - } else { - Some(get_vnode_from_row( - pk_prefix, - *vnode_col_idx_in_pk, - &self.vnodes, - )) - } - } - } - } -} - // TODO: GAT-ify this trait or remove this trait #[async_trait::async_trait] pub trait TableIter: Send { @@ -265,88 +100,6 @@ pub fn get_second(arg: Result<(T, U), E>) -> Result { arg.map(|x| x.1) } -/// Get vnode value with `indices` on the given `row`. -pub fn compute_vnode(row: impl Row, indices: &[usize], vnodes: &Bitmap) -> VirtualNode { - assert!(!indices.is_empty()); - let vnode = VirtualNode::compute_row(&row, indices); - check_vnode_is_set(vnode, vnodes); - - tracing::debug!(target: "events::storage::storage_table", "compute vnode: {:?} key {:?} => {}", row, indices, vnode); - - vnode -} - -pub fn get_vnode_from_row(row: impl Row, index: usize, vnodes: &Bitmap) -> VirtualNode { - let vnode = VirtualNode::from_datum(row.datum_at(index)); - check_vnode_is_set(vnode, vnodes); - - tracing::debug!(target: "events::storage::storage_table", "get vnode from row: {:?} vnode column index {:?} => {}", row, index, vnode); - - vnode -} - -impl TableDistribution { - /// Get vnode values with `indices` on the given `chunk`. - /// - /// Vnode of invisible rows will be included. Only the vnode of visible row check if it's accessible - pub fn compute_chunk_vnode(&self, chunk: &DataChunk, pk_indices: &[usize]) -> Vec { - match &self.compute_vnode { - ComputeVnode::Singleton => { - vec![SINGLETON_VNODE; chunk.capacity()] - } - ComputeVnode::DistKeyIndices { - dist_key_in_pk_indices, - } => { - let dist_key_indices = dist_key_in_pk_indices - .iter() - .map(|idx| pk_indices[*idx]) - .collect_vec(); - - VirtualNode::compute_chunk(chunk, &dist_key_indices) - .into_iter() - .zip_eq_fast(chunk.visibility().iter()) - .map(|(vnode, vis)| { - // Ignore the invisible rows. - if vis { - check_vnode_is_set(vnode, &self.vnodes); - } - vnode - }) - .collect() - } - ComputeVnode::VnodeColumnIndex { - vnode_col_idx_in_pk, - } => { - let array: &PrimitiveArray = - chunk.columns()[pk_indices[*vnode_col_idx_in_pk]].as_int16(); - array - .raw_iter() - .zip_eq_fast(array.null_bitmap().iter()) - .zip_eq_fast(chunk.visibility().iter()) - .map(|((vnode, exist), vis)| { - let vnode = VirtualNode::from_scalar(vnode); - if vis { - assert!(exist); - check_vnode_is_set(vnode, &self.vnodes); - } - vnode - }) - .collect_vec() - } - } - } -} - -/// Check whether the given `vnode` is set in the `vnodes` of this table. -fn check_vnode_is_set(vnode: VirtualNode, vnodes: &Bitmap) { - let is_set = vnodes.is_set(vnode.to_index()); - assert!( - is_set, - "vnode {} should not be accessed by this table", - vnode - ); -} - #[derive(Debug)] pub struct KeyedRow> { vnode_prefixed_key: TableKey, diff --git a/src/stream/Cargo.toml b/src/stream/Cargo.toml index 3f983e963784d..60b18f61d6809 100644 --- a/src/stream/Cargo.toml +++ b/src/stream/Cargo.toml @@ -22,6 +22,7 @@ async-trait = "0.1" auto_enums = "0.8" await-tree = { workspace = true } bytes = "1" +cfg-if = "1" delta_btree_map = { path = "../utils/delta_btree_map" } educe = "0.5" either = "1" @@ -37,10 +38,10 @@ governor = { version = "0.6", default-features = false, features = [ hytra = "0.1.2" itertools = "0.12" local_stats_alloc = { path = "../utils/local_stats_alloc" } -lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "cb2d7c7" } +lru = { workspace = true } maplit = "1.0.2" memcomparable = "0.2" -multimap = "0.9" +multimap = "0.10" parking_lot = "0.12" pin-project = "1" prometheus = { version = "0.13", features = ["process"] } @@ -48,12 +49,13 @@ prost = { workspace = true } rand = "0.8" risingwave_common = { workspace = true } risingwave_connector = { workspace = true } +risingwave_dml = { workspace = true } risingwave_expr = { workspace = true } risingwave_hummock_sdk = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } -risingwave_source = { workspace = true } risingwave_storage = { workspace = true } +rw_futures_util = { workspace = true } serde_json = "1" smallvec = "1" static_assertions = "1" diff --git a/src/stream/benches/stream_hash_agg.rs b/src/stream/benches/stream_hash_agg.rs index dc2f88aff22e5..9d1573bd5fc0c 100644 --- a/src/stream/benches/stream_hash_agg.rs +++ b/src/stream/benches/stream_hash_agg.rs @@ -27,7 +27,7 @@ use risingwave_storage::memory::MemoryStateStore; use risingwave_storage::StateStore; use risingwave_stream::executor::test_utils::agg_executor::new_boxed_hash_agg_executor; use risingwave_stream::executor::test_utils::*; -use risingwave_stream::executor::{BoxedExecutor, PkIndices}; +use risingwave_stream::executor::{Executor, PkIndices}; use tokio::runtime::Runtime; risingwave_expr_impl::enable!(); @@ -48,7 +48,7 @@ fn bench_hash_agg(c: &mut Criterion) { /// This aims to mirror `q17`'s aggregator. /// We can include more executor patterns as needed. -fn setup_bench_hash_agg(store: S) -> BoxedExecutor { +fn setup_bench_hash_agg(store: S) -> Executor { // ---- Define hash agg executor parameters ---- let input_data_types = vec![ // to_char(date_time) @@ -120,7 +120,8 @@ fn setup_bench_hash_agg(store: S) -> BoxedExecutor { ); // ---- Create MockSourceExecutor ---- - let (mut tx, source) = MockSource::channel(schema, PkIndices::new()); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, PkIndices::new()); tx.push_barrier(test_epoch(1), false); for chunk in chunks { tx.push_chunk(chunk); @@ -135,7 +136,7 @@ fn setup_bench_hash_agg(store: S) -> BoxedExecutor { block_on(new_boxed_hash_agg_executor( store, - Box::new(source), + source, false, agg_calls, row_count_index, @@ -147,7 +148,7 @@ fn setup_bench_hash_agg(store: S) -> BoxedExecutor { )) } -pub async fn execute_executor(executor: BoxedExecutor) { +pub async fn execute_executor(executor: Executor) { let mut stream = executor.execute(); while let Some(ret) = stream.next().await { _ = black_box(ret.unwrap()); diff --git a/src/stream/clippy.toml b/src/stream/clippy.toml index dd26a976e1ab1..9d73b2ba754ed 100644 --- a/src/stream/clippy.toml +++ b/src/stream/clippy.toml @@ -7,11 +7,7 @@ disallowed-methods = [ { path = "risingwave_expr::expr::Expression::eval_row", reason = "Please use `NonStrictExpression::eval_row_infallible` instead." }, ] -disallowed-types = [ - { path = "risingwave_common::error::ErrorCode", reason = "Please use per-crate error type instead." }, - { path = "risingwave_common::error::RwError", reason = "Please use per-crate error type instead." }, - { path = "risingwave_common::error::Result", reason = "Please use per-crate error type instead." }, -] +disallowed-types = [] doc-valid-idents = [ "RisingWave", diff --git a/src/stream/src/cache/managed_lru.rs b/src/stream/src/cache/managed_lru.rs index 0bfb8415957c9..9773f3fb51bf0 100644 --- a/src/stream/src/cache/managed_lru.rs +++ b/src/stream/src/cache/managed_lru.rs @@ -20,7 +20,7 @@ use std::ops::{Deref, DerefMut}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; -use lru::{DefaultHasher, KeyRef, LruCache}; +use lru::{DefaultHasher, LruCache}; use risingwave_common::estimate_size::EstimateSize; use risingwave_common::metrics::LabelGuardedIntGauge; use risingwave_common::util::epoch::Epoch; @@ -150,12 +150,20 @@ impl(&mut self, k: &Q) -> Option<&V> where - KeyRef: Borrow, + K: Borrow, Q: Hash + Eq + ?Sized, { self.inner.get(k) } + pub fn peek(&self, k: &Q) -> Option<&V> + where + K: Borrow, + Q: Hash + Eq + ?Sized, + { + self.inner.peek(k) + } + pub fn peek_mut(&mut self, k: &K) -> Option> { let v = self.inner.peek_mut(k); v.map(|inner| { @@ -181,7 +189,7 @@ impl(&self, k: &Q) -> bool where - KeyRef: Borrow, + K: Borrow, Q: Hash + Eq + ?Sized, { self.inner.contains(k) diff --git a/src/stream/src/common/log_store_impl/kv_log_store/mod.rs b/src/stream/src/common/log_store_impl/kv_log_store/mod.rs index 7927df54319c9..797d804053b09 100644 --- a/src/stream/src/common/log_store_impl/kv_log_store/mod.rs +++ b/src/stream/src/common/log_store_impl/kv_log_store/mod.rs @@ -32,21 +32,25 @@ use crate::executor::monitor::StreamingMetrics; mod buffer; mod reader; -mod serde; +pub(crate) mod serde; #[cfg(test)] mod test_utils; mod writer; pub(crate) use reader::{REWIND_BACKOFF_FACTOR, REWIND_BASE_DELAY, REWIND_MAX_DELAY}; +use risingwave_common::hash::VirtualNode; +use risingwave_common::row::ArrayVec; +use risingwave_common::types::{DataType, Datum}; +use risingwave_common::util::sort_util::OrderType; -type SeqIdType = i32; +pub(crate) type SeqIdType = i32; type RowOpCodeType = i16; -const FIRST_SEQ_ID: SeqIdType = 0; +pub(crate) const FIRST_SEQ_ID: SeqIdType = 0; /// Readers truncate the offset at the granularity of seq id. /// None `SeqIdType` means that the whole epoch is truncated. -type ReaderTruncationOffsetType = (u64, Option); +pub(crate) type ReaderTruncationOffsetType = (u64, Option); #[derive(Clone)] pub(crate) struct KvLogStoreReadMetrics { @@ -189,6 +193,121 @@ impl FlushInfo { } } +type KvLogStorePkRow = ArrayVec<[Datum; 3]>; + +pub(crate) struct KvLogStorePkInfo { + pub epoch_column_index: usize, + pub row_op_column_index: usize, + pub seq_id_column_index: usize, + pub predefined_columns: &'static [(&'static str, DataType)], + pub pk_orderings: &'static [OrderType], + pub compute_pk: + fn(vnode: VirtualNode, encoded_epoch: i64, seq_id: Option) -> KvLogStorePkRow, +} + +impl KvLogStorePkInfo { + pub fn pk_len(&self) -> usize { + self.pk_orderings.len() + } + + pub fn predefined_column_len(&self) -> usize { + self.predefined_columns.len() + } + + pub fn pk_types(&self) -> Vec { + (0..self.pk_len()) + .map(|i| self.predefined_columns[i].1.clone()) + .collect() + } +} + +#[expect(deprecated)] +pub(crate) use v1::KV_LOG_STORE_V1_INFO; + +mod v1 { + use std::sync::LazyLock; + + use risingwave_common::constants::log_store::v1::{ + EPOCH_COLUMN_INDEX, KV_LOG_STORE_PREDEFINED_COLUMNS, PK_ORDERING, ROW_OP_COLUMN_INDEX, + SEQ_ID_COLUMN_INDEX, + }; + use risingwave_common::hash::VirtualNode; + use risingwave_common::types::ScalarImpl; + + use super::{KvLogStorePkInfo, KvLogStorePkRow}; + use crate::common::log_store_impl::kv_log_store::SeqIdType; + + #[deprecated] + pub(crate) static KV_LOG_STORE_V1_INFO: LazyLock = LazyLock::new(|| { + fn compute_pk( + _vnode: VirtualNode, + encoded_epoch: i64, + seq_id: Option, + ) -> KvLogStorePkRow { + KvLogStorePkRow::from_array_len( + [ + Some(ScalarImpl::Int64(encoded_epoch)), + seq_id.map(ScalarImpl::Int32), + None, + ], + 2, + ) + } + KvLogStorePkInfo { + epoch_column_index: EPOCH_COLUMN_INDEX, + row_op_column_index: ROW_OP_COLUMN_INDEX, + seq_id_column_index: SEQ_ID_COLUMN_INDEX, + predefined_columns: &KV_LOG_STORE_PREDEFINED_COLUMNS[..], + pk_orderings: &PK_ORDERING[..], + compute_pk, + } + }); +} + +pub(crate) use v2::KV_LOG_STORE_V2_INFO; + +/// A new version of log store schema. Compared to v1, the v2 added a new vnode column to the log store pk, +/// becomes `epoch`, `seq_id` and `vnode`. In this way, providing a log store pk, we can get exactly one single row. +/// +/// In v1, dist key is not in pk, and we will get an error in batch query when we try to compute dist key in pk indices. +/// Now in v2, since we add a vnode column in pk, we can set the vnode index in pk correctly, and the batch query can be +/// correctly executed. See for details. +mod v2 { + use std::sync::LazyLock; + + use risingwave_common::constants::log_store::v2::{ + EPOCH_COLUMN_INDEX, KV_LOG_STORE_PREDEFINED_COLUMNS, PK_ORDERING, ROW_OP_COLUMN_INDEX, + SEQ_ID_COLUMN_INDEX, + }; + use risingwave_common::hash::VirtualNode; + use risingwave_common::types::ScalarImpl; + + use super::{KvLogStorePkInfo, KvLogStorePkRow}; + use crate::common::log_store_impl::kv_log_store::SeqIdType; + + pub(crate) static KV_LOG_STORE_V2_INFO: LazyLock = LazyLock::new(|| { + fn compute_pk( + vnode: VirtualNode, + encoded_epoch: i64, + seq_id: Option, + ) -> KvLogStorePkRow { + KvLogStorePkRow::from([ + Some(ScalarImpl::Int64(encoded_epoch)), + seq_id.map(ScalarImpl::Int32), + vnode.to_datum(), + ]) + } + KvLogStorePkInfo { + epoch_column_index: EPOCH_COLUMN_INDEX, + row_op_column_index: ROW_OP_COLUMN_INDEX, + seq_id_column_index: SEQ_ID_COLUMN_INDEX, + predefined_columns: &KV_LOG_STORE_PREDEFINED_COLUMNS[..], + pk_orderings: &PK_ORDERING[..], + compute_pk, + } + }); +} + pub struct KvLogStoreFactory { state_store: S, @@ -201,6 +320,8 @@ pub struct KvLogStoreFactory { metrics: KvLogStoreMetrics, identity: String, + + pk_info: &'static KvLogStorePkInfo, } impl KvLogStoreFactory { @@ -211,6 +332,7 @@ impl KvLogStoreFactory { max_row_count: usize, metrics: KvLogStoreMetrics, identity: impl Into, + pk_info: &'static KvLogStorePkInfo, ) -> Self { Self { state_store, @@ -219,6 +341,7 @@ impl KvLogStoreFactory { max_row_count, metrics, identity: identity.into(), + pk_info, } } } @@ -230,7 +353,7 @@ impl LogStoreFactory for KvLogStoreFactory { async fn build(self) -> (Self::Reader, Self::Writer) { let table_id = TableId::new(self.table_catalog.id); let (pause_tx, pause_rx) = watch::channel(false); - let serde = LogStoreRowSerde::new(&self.table_catalog, self.vnodes); + let serde = LogStoreRowSerde::new(&self.table_catalog, self.vnodes, self.pk_info); let local_state_store = self .state_store .new_local(NewLocalOptions { @@ -296,21 +419,31 @@ mod tests { use crate::common::log_store_impl::kv_log_store::reader::KvLogStoreReader; use crate::common::log_store_impl::kv_log_store::test_utils::{ calculate_vnode_bitmap, check_rows_eq, check_stream_chunk_eq, - gen_multi_vnode_stream_chunks, gen_stream_chunk, gen_test_log_store_table, TEST_DATA_SIZE, + gen_multi_vnode_stream_chunks, gen_stream_chunk_with_info, gen_test_log_store_table, + TEST_DATA_SIZE, + }; + use crate::common::log_store_impl::kv_log_store::{ + KvLogStoreFactory, KvLogStoreMetrics, KvLogStorePkInfo, KV_LOG_STORE_V2_INFO, }; - use crate::common::log_store_impl::kv_log_store::{KvLogStoreFactory, KvLogStoreMetrics}; #[tokio::test] async fn test_basic() { - for count in 0..20 { - test_basic_inner(count * TEST_DATA_SIZE).await + for count in (0..20).step_by(5) { + #[expect(deprecated)] + test_basic_inner( + count * TEST_DATA_SIZE, + &crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO, + ) + .await; + test_basic_inner(count * TEST_DATA_SIZE, &KV_LOG_STORE_V2_INFO).await; } } - async fn test_basic_inner(max_row_count: usize) { + async fn test_basic_inner(max_row_count: usize, pk_info: &'static KvLogStorePkInfo) { + let gen_stream_chunk = |base| gen_stream_chunk_with_info(base, pk_info); let test_env = prepare_hummock_test_env().await; - let table = gen_test_log_store_table(); + let table = gen_test_log_store_table(pk_info); test_env.register_table(table.clone()).await; @@ -325,6 +458,7 @@ mod tests { max_row_count, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; @@ -395,15 +529,22 @@ mod tests { #[tokio::test] async fn test_recovery() { - for count in 0..20 { - test_recovery_inner(count * TEST_DATA_SIZE).await + for count in (0..20).step_by(5) { + #[expect(deprecated)] + test_recovery_inner( + count * TEST_DATA_SIZE, + &crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO, + ) + .await; + test_recovery_inner(count * TEST_DATA_SIZE, &KV_LOG_STORE_V2_INFO).await; } } - async fn test_recovery_inner(max_row_count: usize) { + async fn test_recovery_inner(max_row_count: usize, pk_info: &'static KvLogStorePkInfo) { + let gen_stream_chunk = |base| gen_stream_chunk_with_info(base, pk_info); let test_env = prepare_hummock_test_env().await; - let table = gen_test_log_store_table(); + let table = gen_test_log_store_table(pk_info); test_env.register_table(table.clone()).await; @@ -419,6 +560,7 @@ mod tests { max_row_count, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; @@ -509,8 +651,10 @@ mod tests { .await .unwrap(); + drop(writer); + // Recovery - test_env.storage.clear_shared_buffer().await.unwrap(); + test_env.storage.clear_shared_buffer(epoch2).await; // Rebuild log reader and writer in recovery let factory = KvLogStoreFactory::new( @@ -520,6 +664,7 @@ mod tests { max_row_count, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; writer @@ -571,15 +716,22 @@ mod tests { #[tokio::test] async fn test_truncate() { - for count in 2..10 { - test_truncate_inner(count).await + for count in (2..10).step_by(3) { + #[expect(deprecated)] + test_truncate_inner( + count, + &crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO, + ) + .await; + test_truncate_inner(count, &KV_LOG_STORE_V2_INFO).await; } } - async fn test_truncate_inner(max_row_count: usize) { + async fn test_truncate_inner(max_row_count: usize, pk_info: &'static KvLogStorePkInfo) { + let gen_stream_chunk = |base| gen_stream_chunk_with_info(base, pk_info); let test_env = prepare_hummock_test_env().await; - let table = gen_test_log_store_table(); + let table = gen_test_log_store_table(pk_info); test_env.register_table(table.clone()).await; @@ -603,6 +755,7 @@ mod tests { max_row_count, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; @@ -717,8 +870,10 @@ mod tests { .await .unwrap(); + drop(writer); + // Recovery - test_env.storage.clear_shared_buffer().await.unwrap(); + test_env.storage.clear_shared_buffer(epoch2).await; // Rebuild log reader and writer in recovery let factory = KvLogStoreFactory::new( @@ -728,6 +883,7 @@ mod tests { max_row_count, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; @@ -796,9 +952,10 @@ mod tests { #[tokio::test] async fn test_update_vnode_recover() { + let pk_info: &'static KvLogStorePkInfo = &KV_LOG_STORE_V2_INFO; let test_env = prepare_hummock_test_env().await; - let table = gen_test_log_store_table(); + let table = gen_test_log_store_table(pk_info); test_env.register_table(table.clone()).await; @@ -820,6 +977,7 @@ mod tests { 10 * TEST_DATA_SIZE, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let factory2 = KvLogStoreFactory::new( test_env.storage.clone(), @@ -828,6 +986,7 @@ mod tests { 10 * TEST_DATA_SIZE, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader1, mut writer1) = factory1.build().await; let (mut reader2, mut writer2) = factory2.build().await; @@ -848,7 +1007,7 @@ mod tests { .unwrap(); reader1.init().await.unwrap(); reader2.init().await.unwrap(); - let [chunk1_1, chunk1_2] = gen_multi_vnode_stream_chunks::<2>(0, 100); + let [chunk1_1, chunk1_2] = gen_multi_vnode_stream_chunks::<2>(0, 100, pk_info); writer1.write_chunk(chunk1_1.clone()).await.unwrap(); writer2.write_chunk(chunk1_2.clone()).await.unwrap(); let epoch2 = test_epoch( @@ -861,7 +1020,7 @@ mod tests { ); writer1.flush_current_epoch(epoch2, false).await.unwrap(); writer2.flush_current_epoch(epoch2, false).await.unwrap(); - let [chunk2_1, chunk2_2] = gen_multi_vnode_stream_chunks::<2>(200, 100); + let [chunk2_1, chunk2_2] = gen_multi_vnode_stream_chunks::<2>(200, 100, pk_info); writer1.write_chunk(chunk2_1.clone()).await.unwrap(); writer2.write_chunk(chunk2_2.clone()).await.unwrap(); @@ -951,8 +1110,11 @@ mod tests { .await .unwrap(); + drop(writer1); + drop(writer2); + // Recovery - test_env.storage.clear_shared_buffer().await.unwrap(); + test_env.storage.clear_shared_buffer(epoch2).await; let vnodes = build_bitmap(0..VirtualNode::COUNT); let factory = KvLogStoreFactory::new( @@ -962,6 +1124,7 @@ mod tests { 10 * TEST_DATA_SIZE, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; writer @@ -1004,9 +1167,11 @@ mod tests { #[tokio::test] async fn test_cancellation_safe() { + let pk_info: &'static KvLogStorePkInfo = &KV_LOG_STORE_V2_INFO; + let gen_stream_chunk = |base| gen_stream_chunk_with_info(base, pk_info); let test_env = prepare_hummock_test_env().await; - let table = gen_test_log_store_table(); + let table = gen_test_log_store_table(pk_info); test_env.register_table(table.clone()).await; @@ -1021,6 +1186,7 @@ mod tests { 0, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; @@ -1078,9 +1244,11 @@ mod tests { #[tokio::test] async fn test_rewind_on_consuming_persisted_log() { + let pk_info: &'static KvLogStorePkInfo = &KV_LOG_STORE_V2_INFO; + let gen_stream_chunk = |base| gen_stream_chunk_with_info(base, pk_info); let test_env = prepare_hummock_test_env().await; - let table = gen_test_log_store_table(); + let table = gen_test_log_store_table(pk_info); test_env.register_table(table.clone()).await; @@ -1162,6 +1330,7 @@ mod tests { 1024, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; @@ -1278,6 +1447,7 @@ mod tests { 1024, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; @@ -1340,6 +1510,7 @@ mod tests { 1024, KvLogStoreMetrics::for_test(), "test", + pk_info, ); let (mut reader, mut writer) = factory.build().await; diff --git a/src/stream/src/common/log_store_impl/kv_log_store/reader.rs b/src/stream/src/common/log_store_impl/kv_log_store/reader.rs index fc90fe64028c8..579d403e98979 100644 --- a/src/stream/src/common/log_store_impl/kv_log_store/reader.rs +++ b/src/stream/src/common/log_store_impl/kv_log_store/reader.rs @@ -190,12 +190,12 @@ impl KvLogStoreReader { // start from the next epoch of last_persisted_epoch Included( self.serde - .serialize_epoch(last_persisted_epoch.next_epoch()), + .serialize_pk_epoch_prefix(last_persisted_epoch.next_epoch()), ) } else { Unbounded }; - let range_end = self.serde.serialize_epoch( + let range_end = self.serde.serialize_pk_epoch_prefix( self.first_write_epoch .expect("should have set first write epoch"), ); diff --git a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs index f76df7f7f304a..3d487e2dd9815 100644 --- a/src/stream/src/common/log_store_impl/kv_log_store/serde.rs +++ b/src/stream/src/common/log_store_impl/kv_log_store/serde.rs @@ -25,17 +25,12 @@ use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::ColumnDesc; -use risingwave_common::constants::log_store::{ - EPOCH_COLUMN_INDEX, EPOCH_COLUMN_TYPE, KV_LOG_STORE_PREDEFINED_COLUMNS, PK_TYPES, - ROW_OP_COLUMN_INDEX, SEQ_ID_COLUMN_INDEX, -}; use risingwave_common::estimate_size::EstimateSize; use risingwave_common::hash::VirtualNode; use risingwave_common::row::{OwnedRow, Row, RowExt}; use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; use risingwave_common::util::row_serde::OrderedRowSerde; -use risingwave_common::util::sort_util::OrderType; use risingwave_common::util::value_encoding::{ BasicSerde, ValueRowDeserializer, ValueRowSerializer, }; @@ -50,7 +45,7 @@ use risingwave_storage::store::StateStoreReadIterStream; use risingwave_storage::table::{compute_vnode, TableDistribution, SINGLETON_VNODE}; use crate::common::log_store_impl::kv_log_store::{ - KvLogStoreReadMetrics, ReaderTruncationOffsetType, RowOpCodeType, SeqIdType, + KvLogStorePkInfo, KvLogStoreReadMetrics, ReaderTruncationOffsetType, RowOpCodeType, SeqIdType, }; const INSERT_OP_CODE: RowOpCodeType = 1; @@ -117,19 +112,30 @@ pub(crate) struct LogStoreRowSerde { /// The schema of payload payload_schema: Vec, + + pk_info: &'static KvLogStorePkInfo, } impl LogStoreRowSerde { - pub(crate) fn new(table_catalog: &Table, vnodes: Option>) -> Self { + pub(crate) fn new( + table_catalog: &Table, + vnodes: Option>, + pk_info: &'static KvLogStorePkInfo, + ) -> Self { let table_columns: Vec = table_catalog .columns .iter() .map(|col| col.column_desc.as_ref().unwrap().into()) .collect(); + let predefined_column_len: usize = pk_info.predefined_column_len(); let dist_key_indices: Vec = table_catalog .distribution_key .iter() - .map(|dist_index| *dist_index as usize) + .map(|dist_index| { + let index = *dist_index as usize; + assert!(index >= predefined_column_len); + index + }) .collect(); let input_value_indices = table_catalog @@ -143,13 +149,15 @@ impl LogStoreRowSerde { .map(|idx| table_columns[*idx].data_type.clone()) .collect_vec(); - // There are 3 predefined columns for kv log store: - assert!(data_types.len() > KV_LOG_STORE_PREDEFINED_COLUMNS.len()); - for i in 0..KV_LOG_STORE_PREDEFINED_COLUMNS.len() { - assert_eq!(data_types[i], KV_LOG_STORE_PREDEFINED_COLUMNS[i].1); + for (schema_data_type, (_, log_store_data_type)) in data_types + .iter() + .take(predefined_column_len) + .zip_eq(pk_info.predefined_columns.iter()) + { + assert_eq!(schema_data_type, log_store_data_type); } - let payload_schema = data_types[KV_LOG_STORE_PREDEFINED_COLUMNS.len()..].to_vec(); + let payload_schema = data_types[predefined_column_len..].to_vec(); let row_serde = BasicSerde::new(input_value_indices.into(), table_columns.into()); @@ -161,13 +169,13 @@ impl LogStoreRowSerde { // epoch and seq_id. The seq_id of barrier is set null, and therefore the second order type // is nulls last - let pk_serde = OrderedRowSerde::new( - Vec::from(PK_TYPES), - vec![OrderType::ascending(), OrderType::ascending_nulls_last()], - ); + let pk_serde = OrderedRowSerde::new(pk_info.pk_types(), Vec::from(pk_info.pk_orderings)); - let epoch_serde = - OrderedRowSerde::new(vec![EPOCH_COLUMN_TYPE], vec![OrderType::ascending()]); + let epoch_col_idx = pk_info.epoch_column_index; + let epoch_serde = OrderedRowSerde::new( + vec![pk_info.predefined_columns[epoch_col_idx].1.clone()], + vec![pk_info.pk_orderings[epoch_col_idx]], + ); let dist_key_indices = if dist_key_indices.is_empty() { if &vnodes != TableDistribution::singleton_vnode_bitmap_ref() { @@ -188,6 +196,7 @@ impl LogStoreRowSerde { dist_key_indices, vnodes, payload_schema, + pk_info, } } @@ -224,21 +233,18 @@ impl LogStoreRowSerde { op: Op, row: impl Row, ) -> (VirtualNode, TableKey, Bytes) { - let pk = [ - Some(ScalarImpl::Int64(Self::encode_epoch(epoch))), - Some(ScalarImpl::Int32(seq_id)), - ]; + let encoded_epoch = Self::encode_epoch(epoch); + let pk = (self.pk_info.compute_pk)(VirtualNode::ZERO, encoded_epoch, Some(seq_id)); let op_code = match op { Op::Insert => INSERT_OP_CODE, Op::Delete => DELETE_OP_CODE, Op::UpdateDelete => UPDATE_DELETE_OP_CODE, Op::UpdateInsert => UPDATE_INSERT_OP_CODE, }; - let extended_row = pk - .clone() - .chain([Some(ScalarImpl::Int16(op_code))]) - .chain(row); - let vnode = self.compute_vnode(&extended_row); + let extended_row_for_vnode = (&pk).chain([Some(ScalarImpl::Int16(op_code))]).chain(&row); + let vnode = self.compute_vnode(&extended_row_for_vnode); + let pk = (self.pk_info.compute_pk)(vnode, encoded_epoch, Some(seq_id)); + let extended_row = (&pk).chain([Some(ScalarImpl::Int16(op_code))]).chain(&row); let key_bytes = serialize_pk_with_vnode(&pk, &self.pk_serde, vnode); let value_bytes = self.row_serde.serialize(extended_row).into(); (vnode, key_bytes, value_bytes) @@ -250,7 +256,7 @@ impl LogStoreRowSerde { vnode: VirtualNode, is_checkpoint: bool, ) -> (TableKey, Bytes) { - let pk = [Some(ScalarImpl::Int64(Self::encode_epoch(epoch))), None]; + let pk = (self.pk_info.compute_pk)(vnode, Self::encode_epoch(epoch), None); let op_code = if is_checkpoint { CHECKPOINT_BARRIER_OP_CODE @@ -258,8 +264,7 @@ impl LogStoreRowSerde { BARRIER_OP_CODE }; - let extended_row = pk - .clone() + let extended_row = (&pk) .chain([Some(ScalarImpl::Int16(op_code))]) .chain(OwnedRow::new(vec![None; self.payload_schema.len()])); let key_bytes = serialize_pk_with_vnode(&pk, &self.pk_serde, vnode); @@ -267,7 +272,7 @@ impl LogStoreRowSerde { (key_bytes, value_bytes) } - pub(crate) fn serialize_epoch(&self, epoch: u64) -> Bytes { + pub(crate) fn serialize_pk_epoch_prefix(&self, epoch: u64) -> Bytes { serialize_pk( [Some(ScalarImpl::Int64(Self::encode_epoch(epoch)))], &self.epoch_serde, @@ -281,10 +286,7 @@ impl LogStoreRowSerde { seq_id: Option, ) -> TableKey { serialize_pk_with_vnode( - [ - Some(ScalarImpl::Int64(Self::encode_epoch(epoch))), - seq_id.map(ScalarImpl::Int32), - ], + (self.pk_info.compute_pk)(vnode, Self::encode_epoch(epoch), seq_id), &self.pk_serde, vnode, ) @@ -296,10 +298,7 @@ impl LogStoreRowSerde { ) -> Bytes { let (epoch, seq_id) = offset; Bytes::from(next_key(&serialize_pk( - [ - Some(ScalarImpl::Int64(Self::encode_epoch(epoch))), - seq_id.map(ScalarImpl::Int32), - ], + (self.pk_info.compute_pk)(VirtualNode::MAX, Self::encode_epoch(epoch), seq_id), &self.pk_serde, ))) } @@ -309,9 +308,17 @@ impl LogStoreRowSerde { fn deserialize(&self, value_bytes: Bytes) -> LogStoreResult<(u64, LogStoreRowOp)> { let row_data = self.row_serde.deserialize(&value_bytes)?; - let payload_row = OwnedRow::new(row_data[KV_LOG_STORE_PREDEFINED_COLUMNS.len()..].to_vec()); - let epoch = Self::decode_epoch(*row_data[EPOCH_COLUMN_INDEX].as_ref().unwrap().as_int64()); - let row_op_code = *row_data[ROW_OP_COLUMN_INDEX].as_ref().unwrap().as_int16(); + let payload_row = OwnedRow::new(row_data[self.pk_info.predefined_column_len()..].to_vec()); + let epoch = Self::decode_epoch( + *row_data[self.pk_info.epoch_column_index] + .as_ref() + .unwrap() + .as_int64(), + ); + let row_op_code = *row_data[self.pk_info.row_op_column_index] + .as_ref() + .unwrap() + .as_int16(); let op = match row_op_code { INSERT_OP_CODE => LogStoreRowOp::Row { @@ -331,13 +338,13 @@ impl LogStoreRowSerde { row: payload_row, }, BARRIER_OP_CODE => { - assert!(row_data[SEQ_ID_COLUMN_INDEX].is_none()); + assert!(row_data[self.pk_info.seq_id_column_index].is_none()); LogStoreRowOp::Barrier { is_checkpoint: false, } } CHECKPOINT_BARRIER_OP_CODE => { - assert!(row_data[SEQ_ID_COLUMN_INDEX].is_none()); + assert!(row_data[self.pk_info.seq_id_column_index].is_none()); LogStoreRowOp::Barrier { is_checkpoint: true, } @@ -777,17 +784,33 @@ mod tests { use crate::common::log_store_impl::kv_log_store::test_utils::{ check_rows_eq, gen_test_data, gen_test_log_store_table, TEST_TABLE_ID, }; - use crate::common::log_store_impl::kv_log_store::{KvLogStoreReadMetrics, SeqIdType}; + use crate::common::log_store_impl::kv_log_store::{ + KvLogStorePkInfo, KvLogStoreReadMetrics, SeqIdType, KV_LOG_STORE_V2_INFO, + }; const EPOCH0: u64 = 1 << EPOCH_AVAILABLE_BITS; const EPOCH1: u64 = EPOCH0 + EPOCH_INC_MIN_STEP_FOR_TEST; const EPOCH2: u64 = EPOCH1 + EPOCH_INC_MIN_STEP_FOR_TEST; #[test] - fn test_serde() { - let table = gen_test_log_store_table(); + fn test_serde_v1() { + #[expect(deprecated)] + test_serde_inner(&crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO); + } + + #[test] + fn test_serde_v2() { + test_serde_inner(&KV_LOG_STORE_V2_INFO); + } + + fn test_serde_inner(pk_info: &'static KvLogStorePkInfo) { + let table = gen_test_log_store_table(pk_info); - let serde = LogStoreRowSerde::new(&table, Some(Arc::new(Bitmap::ones(VirtualNode::COUNT)))); + let serde = LogStoreRowSerde::new( + &table, + Some(Arc::new(Bitmap::ones(VirtualNode::COUNT))), + pk_info, + ); let (ops, rows) = gen_test_data(0); @@ -911,9 +934,26 @@ mod tests { } #[tokio::test] - async fn test_deserialize_stream_chunk() { - let table = gen_test_log_store_table(); - let serde = LogStoreRowSerde::new(&table, Some(Arc::new(Bitmap::ones(VirtualNode::COUNT)))); + async fn test_deserialize_stream_chunk_v1() { + #[expect(deprecated)] + test_deserialize_stream_chunk_inner( + &crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO, + ) + .await + } + + #[tokio::test] + async fn test_deserialize_stream_chunk_v2() { + test_deserialize_stream_chunk_inner(&KV_LOG_STORE_V2_INFO).await + } + + async fn test_deserialize_stream_chunk_inner(pk_info: &'static KvLogStorePkInfo) { + let table = gen_test_log_store_table(pk_info); + let serde = LogStoreRowSerde::new( + &table, + Some(Arc::new(Bitmap::ones(VirtualNode::COUNT))), + pk_info, + ); let (ops, rows) = gen_test_data(0); let mut seq_id = 1; @@ -1048,10 +1088,27 @@ mod tests { } #[tokio::test] - async fn test_row_stream_basic() { - let table = gen_test_log_store_table(); + async fn test_row_stream_basic_v1() { + #[expect(deprecated)] + test_row_stream_basic_inner( + &crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO, + ) + .await + } + + #[tokio::test] + async fn test_row_stream_basic_v2() { + test_row_stream_basic_inner(&KV_LOG_STORE_V2_INFO).await + } - let serde = LogStoreRowSerde::new(&table, Some(Arc::new(Bitmap::ones(VirtualNode::COUNT)))); + async fn test_row_stream_basic_inner(pk_info: &'static KvLogStorePkInfo) { + let table = gen_test_log_store_table(pk_info); + + let serde = LogStoreRowSerde::new( + &table, + Some(Arc::new(Bitmap::ones(VirtualNode::COUNT))), + pk_info, + ); const MERGE_SIZE: usize = 10; @@ -1140,10 +1197,27 @@ mod tests { } #[tokio::test] - async fn test_log_store_stream_basic() { - let table = gen_test_log_store_table(); + async fn test_log_store_stream_basic_v1() { + #[expect(deprecated)] + test_log_store_stream_basic_inner( + &crate::common::log_store_impl::kv_log_store::v1::KV_LOG_STORE_V1_INFO, + ) + .await + } + + #[tokio::test] + async fn test_log_store_stream_basic_v2() { + test_log_store_stream_basic_inner(&KV_LOG_STORE_V2_INFO).await + } - let serde = LogStoreRowSerde::new(&table, Some(Arc::new(Bitmap::ones(VirtualNode::COUNT)))); + async fn test_log_store_stream_basic_inner(pk_info: &'static KvLogStorePkInfo) { + let table = gen_test_log_store_table(pk_info); + + let serde = LogStoreRowSerde::new( + &table, + Some(Arc::new(Bitmap::ones(VirtualNode::COUNT))), + pk_info, + ); let mut seq_id = 1; let (stream, tx1, tx2, ops, rows) = gen_single_test_stream(serde.clone(), &mut seq_id, 0); @@ -1245,9 +1319,14 @@ mod tests { #[tokio::test] async fn test_empty_stream() { - let table = gen_test_log_store_table(); + let pk_info: &'static KvLogStorePkInfo = &KV_LOG_STORE_V2_INFO; + let table = gen_test_log_store_table(pk_info); - let serde = LogStoreRowSerde::new(&table, Some(Arc::new(Bitmap::ones(VirtualNode::COUNT)))); + let serde = LogStoreRowSerde::new( + &table, + Some(Arc::new(Bitmap::ones(VirtualNode::COUNT))), + pk_info, + ); const CHUNK_SIZE: usize = 3; diff --git a/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs b/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs index 36a2e4ffca23b..ef7c39c580982 100644 --- a/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs +++ b/src/stream/src/common/log_store_impl/kv_log_store/test_utils.rs @@ -17,14 +17,13 @@ use rand::RngCore; use risingwave_common::array::{Op, RowRef, StreamChunk}; use risingwave_common::buffer::{Bitmap, BitmapBuilder}; use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId}; -use risingwave_common::constants::log_store::KV_LOG_STORE_PREDEFINED_COLUMNS; use risingwave_common::hash::VirtualNode; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, ScalarImpl, ScalarRef}; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; -use risingwave_common::util::sort_util::OrderType; use risingwave_pb::catalog::PbTable; +use crate::common::log_store_impl::kv_log_store::KvLogStorePkInfo; use crate::common::table::test_utils::gen_prost_table_with_dist_key; pub(crate) const TEST_TABLE_ID: TableId = TableId { table_id: 233 }; @@ -80,27 +79,37 @@ pub(crate) fn gen_sized_test_data(base: i64, max_count: usize) -> (Vec, Vec< (ops, rows) } -pub(crate) fn test_payload_schema() -> Vec { +pub(crate) fn test_payload_schema(pk_info: &'static KvLogStorePkInfo) -> Vec { vec![ - ColumnDesc::unnamed(ColumnId::from(3), DataType::Int64), // id - ColumnDesc::unnamed(ColumnId::from(4), DataType::Varchar), // name + ColumnDesc::unnamed( + ColumnId::from(pk_info.predefined_column_len() as i32), + DataType::Int64, + ), // id + ColumnDesc::unnamed( + ColumnId::from((pk_info.predefined_column_len() + 1) as i32), + DataType::Varchar, + ), // name ] } -pub(crate) fn test_log_store_table_schema() -> Vec { - let mut column_descs = vec![ - ColumnDesc::unnamed(ColumnId::from(0), DataType::Int64), // epoch - ColumnDesc::unnamed(ColumnId::from(1), DataType::Int32), // Seq id - ColumnDesc::unnamed(ColumnId::from(2), DataType::Int16), // op code - ]; - column_descs.extend(test_payload_schema()); +pub(crate) fn test_log_store_table_schema(pk_info: &'static KvLogStorePkInfo) -> Vec { + let mut column_descs = pk_info + .predefined_columns + .iter() + .enumerate() + .map(|(i, (_, data_type))| ColumnDesc::unnamed(ColumnId::from(i as i32), data_type.clone())) + .collect_vec(); + column_descs.extend(test_payload_schema(pk_info)); column_descs } -pub(crate) fn gen_stream_chunk(base: i64) -> StreamChunk { +pub(crate) fn gen_stream_chunk_with_info( + base: i64, + pk_info: &'static KvLogStorePkInfo, +) -> StreamChunk { let (ops, rows) = gen_test_data(base); let mut builder = DataChunkBuilder::new( - test_payload_schema() + test_payload_schema(pk_info) .iter() .map(|col| col.data_type.clone()) .collect_vec(), @@ -116,13 +125,14 @@ pub(crate) fn gen_stream_chunk(base: i64) -> StreamChunk { pub(crate) fn gen_multi_vnode_stream_chunks( base: i64, max_count: usize, + pk_info: &'static KvLogStorePkInfo, ) -> [StreamChunk; MOD_COUNT] { let mut data_builder = (0..MOD_COUNT) .map(|_| { ( Vec::new() as Vec, DataChunkBuilder::new( - test_payload_schema() + test_payload_schema(pk_info) .iter() .map(|col| col.data_type.clone()) .collect_vec(), @@ -149,10 +159,10 @@ pub(crate) fn gen_multi_vnode_stream_chunks( pub(crate) const TEST_SCHEMA_DIST_KEY_INDEX: usize = 0; -pub(crate) fn gen_test_log_store_table() -> PbTable { - let schema = test_log_store_table_schema(); - let order_types = vec![OrderType::ascending(), OrderType::ascending_nulls_last()]; - let pk_index = vec![0_usize, 1_usize]; +pub(crate) fn gen_test_log_store_table(pk_info: &'static KvLogStorePkInfo) -> PbTable { + let schema = test_log_store_table_schema(pk_info); + let order_types = pk_info.pk_orderings.to_vec(); + let pk_index = (0..pk_info.pk_len()).collect(); let read_prefix_len_hint = 0; gen_prost_table_with_dist_key( TEST_TABLE_ID, @@ -160,7 +170,7 @@ pub(crate) fn gen_test_log_store_table() -> PbTable { order_types, pk_index, read_prefix_len_hint, - vec![TEST_SCHEMA_DIST_KEY_INDEX + KV_LOG_STORE_PREDEFINED_COLUMNS.len()], // id field + vec![pk_info.predefined_column_len()], ) } diff --git a/src/stream/src/common/log_store_impl/kv_log_store/writer.rs b/src/stream/src/common/log_store_impl/kv_log_store/writer.rs index a5f002bd73b2a..b4f54d17ed1d1 100644 --- a/src/stream/src/common/log_store_impl/kv_log_store/writer.rs +++ b/src/stream/src/common/log_store_impl/kv_log_store/writer.rs @@ -46,7 +46,9 @@ pub struct KvLogStoreWriter { metrics: KvLogStoreMetrics, - is_paused: watch::Sender, + paused_notifier: watch::Sender, + + is_paused: bool, identity: String, } @@ -58,7 +60,7 @@ impl KvLogStoreWriter { serde: LogStoreRowSerde, tx: LogStoreBufferSender, metrics: KvLogStoreMetrics, - is_paused: watch::Sender, + paused_notifier: watch::Sender, identity: String, ) -> Self { Self { @@ -68,8 +70,9 @@ impl KvLogStoreWriter { serde, tx, metrics, - is_paused, + paused_notifier, identity, + is_paused: false, } } } @@ -81,7 +84,7 @@ impl LogWriter for KvLogStoreWriter { pause_read_on_bootstrap: bool, ) -> LogStoreResult<()> { self.state_store - .init(InitOptions::new_with_epoch(epoch)) + .init(InitOptions::new(epoch, self.serde.vnodes().clone())) .await?; if pause_read_on_bootstrap { self.pause()?; @@ -93,6 +96,9 @@ impl LogWriter for KvLogStoreWriter { } async fn write_chunk(&mut self, chunk: StreamChunk) -> LogStoreResult<()> { + // No data is expected when the stream is paused. + assert!(!self.is_paused); + if chunk.cardinality() == 0 { return Ok(()); } @@ -117,7 +123,7 @@ impl LogWriter for KvLogStoreWriter { self.state_store.insert(key, value, None)?; } flush_info.report(&self.metrics); - self.state_store.flush(Vec::new()).await?; + self.state_store.flush().await?; let vnode_bitmap = vnode_bitmap_builder.finish(); self.tx @@ -133,10 +139,15 @@ impl LogWriter for KvLogStoreWriter { ) -> LogStoreResult<()> { let epoch = self.state_store.epoch(); let mut flush_info = FlushInfo::new(); - for vnode in self.serde.vnodes().iter_vnodes() { - let (key, value) = self.serde.serialize_barrier(epoch, vnode, is_checkpoint); - flush_info.flush_one(key.estimated_size() + value.estimated_size()); - self.state_store.insert(key, value, None)?; + + // When the stream is paused, donot flush barrier to ensure there is no dirty data in state store. + // Besides, barrier on a paused stream is useless in log store because it won't change the log store state. + if !self.is_paused { + for vnode in self.serde.vnodes().iter_vnodes() { + let (key, value) = self.serde.serialize_barrier(epoch, vnode, is_checkpoint); + flush_info.flush_one(key.estimated_size() + value.estimated_size()); + self.state_store.insert(key, value, None)?; + } } self.tx .flush_all_unflushed(|chunk, epoch, start_seq_id, end_seq_id| { @@ -149,20 +160,30 @@ impl LogWriter for KvLogStoreWriter { } Ok(()) })?; + + // No data is expected when the stream is paused. + if self.is_paused { + assert_eq!(flush_info.flush_count, 0); + assert_eq!(flush_info.flush_size, 0); + assert!(!self.state_store.is_dirty()); + } flush_info.report(&self.metrics); - let mut watermark = None; - if let Some(truncation_offset) = self.tx.pop_truncation(epoch) { - watermark = Some(VnodeWatermark::new( + + let watermark = self.tx.pop_truncation(epoch).map(|truncation_offset| { + VnodeWatermark::new( self.serde.vnodes().clone(), self.serde .serialize_truncation_offset_watermark(truncation_offset), - )); - } - self.state_store.flush(vec![]).await?; + ) + }); + self.state_store.flush().await?; let watermark = watermark.into_iter().collect_vec(); self.state_store.seal_current_epoch( next_epoch, - SealCurrentEpochOptions::new(watermark, WatermarkDirection::Ascending), + SealCurrentEpochOptions { + table_watermarks: Some((WatermarkDirection::Ascending, watermark)), + switch_op_consistency_level: None, + }, ); self.tx.barrier(epoch, is_checkpoint, next_epoch); self.seq_id = FIRST_SEQ_ID; @@ -171,20 +192,25 @@ impl LogWriter for KvLogStoreWriter { async fn update_vnode_bitmap(&mut self, new_vnodes: Arc) -> LogStoreResult<()> { self.serde.update_vnode_bitmap(new_vnodes.clone()); + self.state_store.update_vnode_bitmap(new_vnodes.clone()); self.tx.update_vnode(self.state_store.epoch(), new_vnodes); Ok(()) } fn pause(&mut self) -> LogStoreResult<()> { info!("KvLogStore of {} is paused", self.identity); - self.is_paused + assert!(!self.is_paused); + self.is_paused = true; + self.paused_notifier .send(true) .map_err(|_| anyhow!("unable to set pause")) } fn resume(&mut self) -> LogStoreResult<()> { info!("KvLogStore of {} is resumed", self.identity); - self.is_paused + assert!(self.is_paused); + self.is_paused = false; + self.paused_notifier .send(false) .map_err(|_| anyhow!("unable to set resume")) } diff --git a/src/stream/src/common/log_store_impl/mod.rs b/src/stream/src/common/log_store_impl/mod.rs index bd600fc80086f..bcd15ab12a7da 100644 --- a/src/stream/src/common/log_store_impl/mod.rs +++ b/src/stream/src/common/log_store_impl/mod.rs @@ -14,3 +14,4 @@ pub mod in_mem; pub mod kv_log_store; +pub mod subscription_log_store; diff --git a/src/stream/src/common/log_store_impl/subscription_log_store.rs b/src/stream/src/common/log_store_impl/subscription_log_store.rs new file mode 100644 index 0000000000000..999400771a30c --- /dev/null +++ b/src/stream/src/common/log_store_impl/subscription_log_store.rs @@ -0,0 +1,122 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::sync::Arc; + +use itertools::Itertools; +use risingwave_common::array::StreamChunk; +use risingwave_common::buffer::Bitmap; +use risingwave_common::catalog::TableId; +use risingwave_common::hash::VnodeBitmapExt; +use risingwave_connector::sink::log_store::LogStoreResult; +use risingwave_hummock_sdk::table_watermark::{VnodeWatermark, WatermarkDirection}; +use risingwave_storage::store::{InitOptions, LocalStateStore, SealCurrentEpochOptions}; + +use super::kv_log_store::ReaderTruncationOffsetType; +use crate::common::log_store_impl::kv_log_store::serde::LogStoreRowSerde; +use crate::common::log_store_impl::kv_log_store::{SeqIdType, FIRST_SEQ_ID}; + +pub struct SubscriptionLogStoreWriter { + _table_id: TableId, + + seq_id: SeqIdType, + + state_store: LS, + + serde: LogStoreRowSerde, + + _identity: String, +} + +impl SubscriptionLogStoreWriter { + pub(crate) fn new( + table_id: TableId, + state_store: LS, + serde: LogStoreRowSerde, + identity: String, + ) -> Self { + Self { + _table_id: table_id, + seq_id: FIRST_SEQ_ID, + state_store, + serde, + _identity: identity, + } + } + + pub async fn init( + &mut self, + epoch: risingwave_common::util::epoch::EpochPair, + _pause_read_on_bootstrap: bool, + ) -> LogStoreResult<()> { + self.state_store + .init(InitOptions::new(epoch, self.serde.vnodes().clone())) + .await?; + self.seq_id = FIRST_SEQ_ID; + Ok(()) + } + + pub fn write_chunk(&mut self, chunk: StreamChunk) -> LogStoreResult<()> { + if chunk.cardinality() == 0 { + return Ok(()); + } + let epoch = self.state_store.epoch(); + let start_seq_id = self.seq_id; + self.seq_id += chunk.cardinality() as SeqIdType; + for (i, (op, row)) in chunk.rows().enumerate() { + let seq_id = start_seq_id + (i as SeqIdType); + let (_vnode, key, value) = self.serde.serialize_data_row(epoch, seq_id, op, row); + self.state_store.insert(key, value, None)?; + } + Ok(()) + } + + pub async fn flush_current_epoch( + &mut self, + next_epoch: u64, + is_checkpoint: bool, + truncate_offset: Option, + ) -> LogStoreResult<()> { + let epoch = self.state_store.epoch(); + for vnode in self.serde.vnodes().iter_vnodes() { + let (key, value) = self.serde.serialize_barrier(epoch, vnode, is_checkpoint); + self.state_store.insert(key, value, None)?; + } + + let watermark = truncate_offset.map(|truncate_offset| { + VnodeWatermark::new( + self.serde.vnodes().clone(), + self.serde + .serialize_truncation_offset_watermark(truncate_offset), + ) + }); + self.state_store.flush().await?; + let watermark = watermark.into_iter().collect_vec(); + self.state_store.seal_current_epoch( + next_epoch, + SealCurrentEpochOptions { + table_watermarks: Some((WatermarkDirection::Ascending, watermark)), + switch_op_consistency_level: None, + }, + ); + self.seq_id = FIRST_SEQ_ID; + Ok(()) + } + + pub fn update_vnode_bitmap(&mut self, new_vnodes: Arc) -> LogStoreResult<()> { + self.serde.update_vnode_bitmap(new_vnodes.clone()); + self.state_store.update_vnode_bitmap(new_vnodes); + Ok(()) + } +} diff --git a/src/stream/src/common/mod.rs b/src/stream/src/common/mod.rs index 18129884db16c..c12fb5de1fbab 100644 --- a/src/stream/src/common/mod.rs +++ b/src/stream/src/common/mod.rs @@ -12,10 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -pub use builder::*; pub use column_mapping::*; +pub use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder; -mod builder; pub mod cache; mod column_mapping; pub mod log_store_impl; diff --git a/src/stream/src/common/table/state_table.rs b/src/stream/src/common/table/state_table.rs index 269e1dd0490fc..87f61d4874074 100644 --- a/src/stream/src/common/table/state_table.rs +++ b/src/stream/src/common/table/state_table.rs @@ -60,6 +60,7 @@ use risingwave_storage::store::{ use risingwave_storage::table::merge_sort::merge_sort; use risingwave_storage::table::{KeyedRow, TableDistribution}; use risingwave_storage::StateStore; +use thiserror_ext::AsReport; use tracing::{trace, Instrument}; use super::watermark::{WatermarkBufferByEpoch, WatermarkBufferStrategy}; @@ -149,6 +150,8 @@ pub struct StateTableInner< /// 1. Computing output_value_indices to ser/de replicated rows. /// 2. Computing output pk indices to used them for backfill state. output_indices: Vec, + + is_consistent_op: bool, } /// `StateTable` will use `BasicSerde` as default @@ -175,7 +178,7 @@ where /// as it needs to wait for prev epoch to be committed. pub async fn init_epoch(&mut self, epoch: EpochPair) -> StorageResult<()> { self.local_store - .init(InitOptions::new_with_epoch(epoch)) + .init(InitOptions::new(epoch, self.vnodes().clone())) .await } } @@ -192,7 +195,7 @@ where /// No need to `wait_for_epoch`, so it should complete immediately. pub fn init_epoch(&mut self, epoch: EpochPair) { self.local_store - .init(InitOptions::new_with_epoch(epoch)) + .init(InitOptions::new(epoch, self.vnodes().clone())) .now_or_never() .expect("non-replicated state store should start immediately.") .expect("non-replicated state store should not wait_for_epoch, and fail because of it.") @@ -207,14 +210,14 @@ fn consistent_old_value_op(row_serde: impl ValueRowSerde) -> OpConsistencyLevel let first = match row_serde.deserialize(first) { Ok(rows) => rows, Err(e) => { - error!(err = %e, value = ?first, "fail to deserialize serialized value"); + error!(error = %e.as_report(), value = ?first, "fail to deserialize serialized value"); return false; } }; let second = match row_serde.deserialize(second) { Ok(rows) => rows, Err(e) => { - error!(err = %e, value = ?second, "fail to deserialize serialized value"); + error!(error = %e.as_report(), value = ?second, "fail to deserialize serialized value"); return false; } }; @@ -300,7 +303,7 @@ where // FIXME(yuhao): only use `dist_key_in_pk` in the proto let dist_key_in_pk_indices = if table_catalog.get_dist_key_in_pk().is_empty() { - get_dist_key_in_pk_indices(&dist_key_indices, &pk_indices) + get_dist_key_in_pk_indices(&dist_key_indices, &pk_indices).unwrap() } else { table_catalog .get_dist_key_in_pk() @@ -354,7 +357,7 @@ where OpConsistencyLevel::Inconsistent }; - let table_option = TableOption::build_table_option(table_catalog.get_properties()); + let table_option = TableOption::new(table_catalog.retention_seconds); let new_local_options = if IS_REPLICATED { NewLocalOptions::new_replicated(table_id, op_consistency_level, table_option) } else { @@ -425,6 +428,7 @@ where data_types, output_indices, i2o_mapping, + is_consistent_op, } } @@ -604,6 +608,7 @@ where data_types, output_indices: vec![], i2o_mapping: ColIndexMapping::new(vec![], 0), + is_consistent_op, } } @@ -664,6 +669,10 @@ where fn is_dirty(&self) -> bool { self.local_store.is_dirty() || self.state_clean_watermark.is_some() } + + pub fn is_consistent_op(&self) -> bool { + self.is_consistent_op + } } impl StateTableInner @@ -771,6 +780,13 @@ where !self.is_dirty(), "vnode bitmap should only be updated when state table is clean" ); + let prev_vnodes = self.local_store.update_vnode_bitmap(new_vnodes.clone()); + assert_eq!( + &prev_vnodes, + self.vnodes(), + "state table and state store vnode bitmap mismatches" + ); + if self.distribution.is_singleton() { assert_eq!( &new_vnodes, @@ -1025,7 +1041,24 @@ where } pub async fn commit(&mut self, new_epoch: EpochPair) -> StreamExecutorResult<()> { + self.commit_with_switch_consistent_op(new_epoch, None).await + } + + pub async fn commit_with_switch_consistent_op( + &mut self, + new_epoch: EpochPair, + switch_consistent_op: Option, + ) -> StreamExecutorResult<()> { assert_eq!(self.epoch(), new_epoch.prev); + let switch_op_consistency_level = switch_consistent_op.map(|enable_consistent_op| { + assert_ne!(self.is_consistent_op, enable_consistent_op); + self.is_consistent_op = enable_consistent_op; + if enable_consistent_op { + consistent_old_value_op(self.row_serde.clone()) + } else { + OpConsistencyLevel::Inconsistent + } + }); trace!( table_id = %self.table_id, epoch = ?self.epoch(), @@ -1036,11 +1069,16 @@ where self.watermark_buffer_strategy.tick(); if !self.is_dirty() { // If the state table is not modified, go fast path. - self.local_store - .seal_current_epoch(new_epoch.curr, SealCurrentEpochOptions::no_watermark()); + self.local_store.seal_current_epoch( + new_epoch.curr, + SealCurrentEpochOptions { + table_watermarks: None, + switch_op_consistency_level, + }, + ); return Ok(()); } else { - self.seal_current_epoch(new_epoch.curr) + self.seal_current_epoch(new_epoch.curr, switch_op_consistency_level) .instrument(tracing::info_span!("state_table_commit")) .await?; } @@ -1096,21 +1134,12 @@ where Ok(()) } - // TODO(st1page): maybe we should extract a pub struct to do it - /// just specially used by those state table read-only and after the call the data - /// in the epoch will be visible - pub fn commit_no_data_expected(&mut self, new_epoch: EpochPair) { - assert_eq!(self.epoch(), new_epoch.prev); - assert!(!self.is_dirty()); - // Tick the watermark buffer here because state table is expected to be committed once - // per epoch. - self.watermark_buffer_strategy.tick(); - self.local_store - .seal_current_epoch(new_epoch.curr, SealCurrentEpochOptions::no_watermark()); - } - /// Write to state store. - async fn seal_current_epoch(&mut self, next_epoch: u64) -> StreamExecutorResult<()> { + async fn seal_current_epoch( + &mut self, + next_epoch: u64, + switch_op_consistency_level: Option, + ) -> StreamExecutorResult<()> { let watermark = self.state_clean_watermark.take(); watermark.as_ref().inspect(|watermark| { trace!(table_id = %self.table_id, watermark = ?watermark, "state cleaning"); @@ -1196,14 +1225,17 @@ where self.watermark_cache.clear(); } - self.local_store.flush(vec![]).await?; - let seal_opt = match seal_watermark { - Some((direction, watermark)) => { - SealCurrentEpochOptions::new(vec![watermark], direction) - } - None => SealCurrentEpochOptions::no_watermark(), - }; - self.local_store.seal_current_epoch(next_epoch, seal_opt); + self.local_store.flush().await?; + let table_watermarks = + seal_watermark.map(|(direction, watermark)| (direction, vec![watermark])); + + self.local_store.seal_current_epoch( + next_epoch, + SealCurrentEpochOptions { + table_watermarks, + switch_op_consistency_level, + }, + ); Ok(()) } diff --git a/src/stream/src/common/table/state_table_cache.rs b/src/stream/src/common/table/state_table_cache.rs index 06acd7e710dce..8c8014548a1e1 100644 --- a/src/stream/src/common/table/state_table_cache.rs +++ b/src/stream/src/common/table/state_table_cache.rs @@ -453,23 +453,23 @@ mod tests { #[test] fn test_watermark_cache_syncing() { - let v1 = vec![ + let v1 = [ Some(Timestamptz::from_secs(1000).unwrap().to_scalar_value()), Some(1000i64.into()), ]; - let v2 = vec![ + let v2 = [ Some(Timestamptz::from_secs(3000).unwrap().to_scalar_value()), Some(1000i64.into()), ]; - let v3 = vec![ + let v3 = [ Some(Timestamptz::from_secs(2000).unwrap().to_scalar_value()), Some(1000i64.into()), ]; let mut cache = StateTableWatermarkCache::new(3); let mut filler = cache.begin_syncing(); - filler.insert_unchecked(DefaultOrdered(v1.into_owned_row()), ()); - filler.insert_unchecked(DefaultOrdered(v2.into_owned_row()), ()); - filler.insert_unchecked(DefaultOrdered(v3.into_owned_row()), ()); + filler.insert_unchecked(DefaultOrdered(v1.to_owned_row()), ()); + filler.insert_unchecked(DefaultOrdered(v2.to_owned_row()), ()); + filler.insert_unchecked(DefaultOrdered(v3.to_owned_row()), ()); filler.finish(); assert_eq!(cache.len(), 3); assert_eq!( diff --git a/src/stream/src/executor/actor.rs b/src/stream/src/executor/actor.rs index ffc29c2d25dac..3380dd5c1a9f7 100644 --- a/src/stream/src/executor/actor.rs +++ b/src/stream/src/executor/actor.rs @@ -24,9 +24,10 @@ use risingwave_common::error::ErrorSuppressor; use risingwave_common::log::LogSuppresser; use risingwave_common::metrics::GLOBAL_ERROR_METRICS; use risingwave_common::util::epoch::EpochPair; -use risingwave_expr::expr_context::expr_context_scope; +use risingwave_expr::expr_context::{expr_context_scope, FRAGMENT_ID}; use risingwave_expr::ExprError; use risingwave_pb::plan_common::ExprContext; +use risingwave_pb::stream_plan::PbStreamActor; use thiserror_ext::AsReport; use tokio_stream::StreamExt; use tracing::Instrument; @@ -35,12 +36,13 @@ use super::monitor::StreamingMetrics; use super::subtask::SubtaskHandle; use super::StreamConsumer; use crate::error::StreamResult; -use crate::task::{ActorId, SharedContext}; +use crate::task::{ActorId, LocalBarrierManager}; /// Shared by all operators of an actor. pub struct ActorContext { pub id: ActorId, pub fragment_id: u32, + pub mview_definition: String, // TODO(eric): these seem to be useless now? last_mem_val: Arc, @@ -49,38 +51,45 @@ pub struct ActorContext { pub streaming_metrics: Arc, pub error_suppressor: Arc>, + + pub dispatch_num: usize, } pub type ActorContextRef = Arc; impl ActorContext { - pub fn create(id: ActorId) -> ActorContextRef { + pub fn for_test(id: ActorId) -> ActorContextRef { Arc::new(Self { id, fragment_id: 0, + mview_definition: "".to_string(), cur_mem_val: Arc::new(0.into()), last_mem_val: Arc::new(0.into()), total_mem_val: Arc::new(TrAdder::new()), streaming_metrics: Arc::new(StreamingMetrics::unused()), error_suppressor: Arc::new(Mutex::new(ErrorSuppressor::new(10))), + // Set 1 for test to enable sanity check on table + dispatch_num: 1, }) } - pub fn create_with_metrics( - id: ActorId, - fragment_id: u32, + pub fn create( + stream_actor: &PbStreamActor, total_mem_val: Arc>, streaming_metrics: Arc, unique_user_errors: usize, + dispatch_num: usize, ) -> ActorContextRef { Arc::new(Self { - id, - fragment_id, + id: stream_actor.actor_id, + fragment_id: stream_actor.fragment_id, + mview_definition: stream_actor.mview_definition.clone(), cur_mem_val: Arc::new(0.into()), last_mem_val: Arc::new(0.into()), total_mem_val, streaming_metrics, error_suppressor: Arc::new(Mutex::new(ErrorSuppressor::new(unique_user_errors))), + dispatch_num, }) } @@ -132,10 +141,10 @@ pub struct Actor { /// The subtasks to execute concurrently. subtasks: Vec, - context: Arc, _metrics: Arc, - actor_context: ActorContextRef, + pub actor_context: ActorContextRef, expr_context: ExprContext, + barrier_manager: LocalBarrierManager, } impl Actor @@ -145,31 +154,34 @@ where pub fn new( consumer: C, subtasks: Vec, - context: Arc, metrics: Arc, actor_context: ActorContextRef, expr_context: ExprContext, + barrier_manager: LocalBarrierManager, ) -> Self { Self { consumer, subtasks, - context, _metrics: metrics, actor_context, expr_context, + barrier_manager, } } #[inline(always)] pub async fn run(mut self) -> StreamResult<()> { - expr_context_scope(self.expr_context.clone(), async move { - tokio::join!( - // Drive the subtasks concurrently. - join_all(std::mem::take(&mut self.subtasks)), - self.run_consumer(), - ) - .1 - }) + FRAGMENT_ID::scope( + self.actor_context.fragment_id, + expr_context_scope(self.expr_context.clone(), async move { + tokio::join!( + // Drive the subtasks concurrently. + join_all(std::mem::take(&mut self.subtasks)), + self.run_consumer(), + ) + .1 + }), + ) .await } @@ -219,7 +231,7 @@ where .into())); // Collect barriers to local barrier manager - self.context.barrier_manager().collect(id, &barrier); + self.barrier_manager.collect(id, &barrier); // Then stop this actor if asked if barrier.is_stop(id) { diff --git a/src/stream/src/executor/agg_common.rs b/src/stream/src/executor/agg_common.rs index 603cb7f244f87..91c414877a76b 100644 --- a/src/stream/src/executor/agg_common.rs +++ b/src/stream/src/executor/agg_common.rs @@ -29,7 +29,7 @@ pub struct AggExecutorArgs { pub version: PbAggNodeVersion, // basic - pub input: Box, + pub input: Executor, pub actor_ctx: ActorContextRef, pub info: ExecutorInfo, diff --git a/src/stream/src/executor/aggregation/distinct.rs b/src/stream/src/executor/aggregation/distinct.rs index f82aa0893f50a..8bb298b47c99d 100644 --- a/src/stream/src/executor/aggregation/distinct.rs +++ b/src/stream/src/executor/aggregation/distinct.rs @@ -392,7 +392,7 @@ mod tests { &agg_calls, Arc::new(AtomicU64::new(0)), &dedup_tables, - ActorContext::create(0), + ActorContext::for_test(0), ); // --- chunk 1 --- @@ -483,7 +483,7 @@ mod tests { &agg_calls, Arc::new(AtomicU64::new(0)), &dedup_tables, - ActorContext::create(0), + ActorContext::for_test(0), ); // --- chunk 3 --- @@ -572,7 +572,7 @@ mod tests { &agg_calls, Arc::new(AtomicU64::new(0)), &dedup_tables, - ActorContext::create(0), + ActorContext::for_test(0), ); let chunk = StreamChunk::from_pretty( diff --git a/src/stream/src/executor/backfill/arrangement_backfill.rs b/src/stream/src/executor/backfill/arrangement_backfill.rs index 6eb0a7efdf783..dbd4a0848a6bd 100644 --- a/src/stream/src/executor/backfill/arrangement_backfill.rs +++ b/src/stream/src/executor/backfill/arrangement_backfill.rs @@ -16,34 +16,36 @@ use std::pin::pin; use std::sync::Arc; use either::Either; -use futures::stream::select_with_strategy; +use futures::stream::{select_all, select_with_strategy}; use futures::{stream, StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use itertools::Itertools; -use risingwave_common::array::{Op, StreamChunk}; +use risingwave_common::array::{DataChunk, Op, StreamChunk}; use risingwave_common::bail; -use risingwave_common::catalog::Schema; use risingwave_common::hash::{VirtualNode, VnodeBitmapExt}; +use risingwave_common::row::OwnedRow; use risingwave_common::util::chunk_coalesce::DataChunkBuilder; -use risingwave_common::util::iter_util::ZipEqDebug; use risingwave_storage::row_serde::value_serde::ValueRowSerde; +use risingwave_storage::store::PrefetchOptions; use risingwave_storage::StateStore; use crate::common::table::state_table::{ReplicatedStateTable, StateTable}; #[cfg(debug_assertions)] use crate::executor::backfill::utils::METADATA_STATE_LEN; use crate::executor::backfill::utils::{ - compute_bounds, create_builder, get_progress_per_vnode, iter_chunks, mapping_chunk, - mapping_message, mark_chunk_ref_by_vnode, owned_row_iter, persist_state_per_vnode, + compute_bounds, create_builder, get_progress_per_vnode, mapping_chunk, mapping_message, + mark_chunk_ref_by_vnode, owned_row_iter, persist_state_per_vnode, update_backfill_metrics, update_pos_by_vnode, BackfillProgressPerVnode, BackfillState, }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, - Message, PkIndicesRef, StreamExecutorError, + expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, HashMap, Message, + StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; +type Builders = HashMap; + /// Similar to [`super::no_shuffle_backfill::BackfillExecutor`]. /// Main differences: /// - [`ArrangementBackfillExecutor`] can reside on a different CN, so it can be scaled @@ -54,7 +56,7 @@ pub struct ArrangementBackfillExecutor { upstream_table: ReplicatedStateTable, /// Upstream with the same schema with the upstream table. - upstream: BoxedExecutor, + upstream: Executor, /// Internal state table for persisting state of backfill state. state_table: StateTable, @@ -66,8 +68,6 @@ pub struct ArrangementBackfillExecutor { actor_id: ActorId, - info: ExecutorInfo, - metrics: Arc, chunk_size: usize, @@ -83,9 +83,8 @@ where #[allow(clippy::too_many_arguments)] #[allow(dead_code)] pub fn new( - info: ExecutorInfo, upstream_table: ReplicatedStateTable, - upstream: BoxedExecutor, + upstream: Executor, state_table: StateTable, output_indices: Vec, progress: CreateMviewProgress, @@ -94,7 +93,6 @@ where rate_limit: Option, ) -> Self { Self { - info, upstream_table, upstream, state_table, @@ -109,6 +107,7 @@ where #[try_stream(ok = Message, error = StreamExecutorError)] async fn execute_inner(mut self) { + tracing::debug!("Arrangement Backfill Executor started"); // The primary key columns, in the output columns of the upstream_table scan. // Table scan scans a subset of the columns of the upstream table. let pk_in_output_indices = self.upstream_table.pk_in_output_indices().unwrap(); @@ -129,22 +128,24 @@ where .iter() .map(|field| field.data_type.clone()) .collect_vec(); - let mut builders = upstream_table + let mut builders: Builders = upstream_table .vnodes() .iter_vnodes() - .map(|_| { - create_builder( + .map(|vnode| { + let builder = create_builder( self.rate_limit, self.chunk_size, snapshot_data_types.clone(), - ) + ); + (vnode, builder) }) - .collect_vec(); + .collect(); let mut upstream = self.upstream.execute(); // Poll the upstream to get the first barrier. let first_barrier = expect_first_barrier(&mut upstream).await?; + let mut paused = first_barrier.is_pause_on_startup(); let first_epoch = first_barrier.epoch; self.state_table.init_epoch(first_barrier.epoch); @@ -182,7 +183,7 @@ where let mut snapshot_read_epoch; // Keep track of rows from the snapshot. - let mut total_snapshot_processed_rows: u64 = 0; + let mut total_snapshot_processed_rows: u64 = backfill_state.get_snapshot_row_count(); // Arrangement Backfill Algorithm: // @@ -216,6 +217,8 @@ where 'backfill_loop: loop { let mut cur_barrier_snapshot_processed_rows: u64 = 0; let mut cur_barrier_upstream_processed_rows: u64 = 0; + let mut snapshot_read_complete = false; + let mut has_snapshot_read = false; // NOTE(kwannoel): Scope it so that immutable reference to `upstream_table` can be // dropped. Then we can write to `upstream_table` on barrier in the @@ -223,22 +226,22 @@ where { let left_upstream = upstream.by_ref().map(Either::Left); - let right_snapshot = pin!(Self::snapshot_read_per_vnode( + let right_snapshot = pin!(Self::make_snapshot_stream( &upstream_table, backfill_state.clone(), // FIXME: Use mutable reference instead. - &mut builders, + paused, ) - .map(Either::Right),); + .map(Either::Right)); // Prefer to select upstream, so we can stop snapshot stream as soon as the // barrier comes. - let backfill_stream = + let mut backfill_stream = select_with_strategy(left_upstream, right_snapshot, |_: &mut ()| { stream::PollNext::Left }); #[for_await] - for either in backfill_stream { + for either in &mut backfill_stream { match either { // Upstream Either::Left(msg) => { @@ -264,8 +267,24 @@ where } // Snapshot read Either::Right(msg) => { + has_snapshot_read = true; match msg? { None => { + // Consume remaining rows in the builder. + for (vnode, builder) in &mut builders { + if let Some(data_chunk) = builder.consume_all() { + yield Self::handle_snapshot_chunk( + data_chunk, + *vnode, + &pk_in_output_indices, + &mut backfill_state, + &mut cur_barrier_snapshot_processed_rows, + &mut total_snapshot_processed_rows, + &self.output_indices, + )?; + } + } + // End of the snapshot read stream. // We should not mark the chunk anymore, // otherwise, we will ignore some rows @@ -274,37 +293,77 @@ where // mark. for chunk in upstream_chunk_buffer.drain(..) { let chunk_cardinality = chunk.cardinality() as u64; - cur_barrier_snapshot_processed_rows += + cur_barrier_upstream_processed_rows += chunk_cardinality; - total_snapshot_processed_rows += chunk_cardinality; yield Message::Chunk(mapping_chunk( chunk, &self.output_indices, )); } - + update_backfill_metrics( + &self.metrics, + self.actor_id, + upstream_table_id, + cur_barrier_snapshot_processed_rows, + cur_barrier_upstream_processed_rows, + ); break 'backfill_loop; } - Some((vnode, chunk)) => { - // Raise the current position. - // As snapshot read streams are ordered by pk, so we can - // just use the last row to update `current_pos`. - update_pos_by_vnode( + Some((vnode, row)) => { + let builder = builders.get_mut(&vnode).unwrap(); + if let Some(chunk) = builder.append_one_row(row) { + yield Self::handle_snapshot_chunk( + chunk, + vnode, + &pk_in_output_indices, + &mut backfill_state, + &mut cur_barrier_snapshot_processed_rows, + &mut total_snapshot_processed_rows, + &self.output_indices, + )?; + } + } + } + } + } + } + + // Before processing barrier, if did not snapshot read, + // do a snapshot read first. + // This is so we don't lose the tombstone iteration progress. + // If paused, we also can't read any snapshot records. + if !has_snapshot_read && !paused { + // If we have not snapshot read, builders must all be empty. + debug_assert!(builders.values().all(|b| b.is_empty())); + let (_, snapshot) = backfill_stream.into_inner(); + #[for_await] + for msg in snapshot { + let Either::Right(msg) = msg else { + bail!("BUG: snapshot_read contains upstream messages"); + }; + match msg? { + None => { + // End of the snapshot read stream. + // We let the barrier handling logic take care of upstream updates. + // But we still want to exit backfill loop, so we mark snapshot read complete. + snapshot_read_complete = true; + break; + } + Some((vnode, row)) => { + let builder = builders.get_mut(&vnode).unwrap(); + if let Some(chunk) = builder.append_one_row(row) { + yield Self::handle_snapshot_chunk( + chunk, vnode, - &chunk, &pk_in_output_indices, &mut backfill_state, - )?; - - let chunk_cardinality = chunk.cardinality() as u64; - cur_barrier_snapshot_processed_rows += chunk_cardinality; - total_snapshot_processed_rows += chunk_cardinality; - let chunk = Message::Chunk(mapping_chunk( - chunk, + &mut cur_barrier_snapshot_processed_rows, + &mut total_snapshot_processed_rows, &self.output_indices, - )); - yield chunk; + )?; } + + break; } } } @@ -321,31 +380,48 @@ where }; // Process barrier: + // - handle mutations // - consume snapshot rows left in builder. // - consume upstream buffer chunk // - switch snapshot + // handle mutations + if let Some(mutation) = barrier.mutation.as_deref() { + use crate::executor::Mutation; + match mutation { + Mutation::Pause => { + paused = true; + } + Mutation::Resume => { + paused = false; + } + _ => (), + } + } + // consume snapshot rows left in builder. // NOTE(kwannoel): `zip_eq_debug` does not work here, // we encounter "higher-ranked lifetime error". - for (vnode, chunk) in vnodes.iter_vnodes().zip_eq(builders.iter_mut().map(|b| { - b.consume_all().map(|chunk| { + for (vnode, chunk) in builders.iter_mut().map(|(vnode, b)| { + let chunk = b.consume_all().map(|chunk| { let ops = vec![Op::Insert; chunk.capacity()]; StreamChunk::from_parts(ops, chunk) - }) - })) { + }); + (vnode, chunk) + }) { if let Some(chunk) = chunk { + let chunk_cardinality = chunk.cardinality() as u64; // Raise the current position. // As snapshot read streams are ordered by pk, so we can // just use the last row to update `current_pos`. update_pos_by_vnode( - vnode, + *vnode, &chunk, &pk_in_output_indices, &mut backfill_state, + chunk_cardinality, )?; - let chunk_cardinality = chunk.cardinality() as u64; cur_barrier_snapshot_processed_rows += chunk_cardinality; total_snapshot_processed_rows += chunk_cardinality; yield Message::Chunk(mapping_chunk(chunk, &self.output_indices)); @@ -353,7 +429,6 @@ where } // consume upstream buffer chunk - let upstream_chunk_buffer_is_empty = upstream_chunk_buffer.is_empty(); for chunk in upstream_chunk_buffer.drain(..) { cur_barrier_upstream_processed_rows += chunk.cardinality() as u64; // FIXME: Replace with `snapshot_is_processed` @@ -376,27 +451,15 @@ where upstream_table.write_chunk(chunk); } - if upstream_chunk_buffer_is_empty { - upstream_table.commit_no_data_expected(barrier.epoch) - } else { - upstream_table.commit(barrier.epoch).await?; - } + upstream_table.commit(barrier.epoch).await?; - self.metrics - .arrangement_backfill_snapshot_read_row_count - .with_label_values(&[ - upstream_table_id.to_string().as_str(), - self.actor_id.to_string().as_str(), - ]) - .inc_by(cur_barrier_snapshot_processed_rows); - - self.metrics - .arrangement_backfill_upstream_output_row_count - .with_label_values(&[ - upstream_table_id.to_string().as_str(), - self.actor_id.to_string().as_str(), - ]) - .inc_by(cur_barrier_upstream_processed_rows); + update_backfill_metrics( + &self.metrics, + self.actor_id, + upstream_table_id, + cur_barrier_snapshot_processed_rows, + cur_barrier_upstream_processed_rows, + ); // Update snapshot read epoch. snapshot_read_epoch = barrier.epoch.prev; @@ -430,6 +493,10 @@ where yield Message::Barrier(barrier); // We will switch snapshot at the start of the next iteration of the backfill loop. + // Unless snapshot read is already completed. + if snapshot_read_complete { + break 'backfill_loop; + } } } @@ -450,59 +517,111 @@ where "backfill_finished_wait_for_barrier" ); // If not finished then we need to update state, otherwise no need. - if let Message::Barrier(barrier) = &msg - && !is_completely_finished - { - // If snapshot was empty, we do not need to backfill, - // but we still need to persist the finished state. - // We currently persist it on the second barrier here rather than first. - // This is because we can't update state table in first epoch, - // since it expects to have been initialized in previous epoch - // (there's no epoch before the first epoch). - for vnode in upstream_table.vnodes().iter_vnodes() { - backfill_state.finish_progress(vnode, upstream_table.pk_indices().len()); - } + if let Message::Barrier(barrier) = &msg { + if is_completely_finished { + // If already finished, no need to persist any state. + } else { + // If snapshot was empty, we do not need to backfill, + // but we still need to persist the finished state. + // We currently persist it on the second barrier here rather than first. + // This is because we can't update state table in first epoch, + // since it expects to have been initialized in previous epoch + // (there's no epoch before the first epoch). + for vnode in upstream_table.vnodes().iter_vnodes() { + backfill_state + .finish_progress(vnode, upstream_table.pk_indices().len()); + } - persist_state_per_vnode( - barrier.epoch, - &mut self.state_table, - &mut backfill_state, - #[cfg(debug_assertions)] - state_len, - vnodes.iter_vnodes(), - ) - .await?; + persist_state_per_vnode( + barrier.epoch, + &mut self.state_table, + &mut backfill_state, + #[cfg(debug_assertions)] + state_len, + vnodes.iter_vnodes(), + ) + .await?; + } self.progress .finish(barrier.epoch.curr, total_snapshot_processed_rows); + tracing::trace!( + epoch = ?barrier.epoch, + "Updated CreateMaterializedTracker" + ); yield msg; break; - } else { - // Allow other messages to pass through. - yield msg; } + // Allow other messages to pass through. + // We won't yield twice here, since if there's a barrier, + // we will always break out of the loop. + yield msg; } } + tracing::trace!( + "Arrangement Backfill has already finished and forward messages directly to the downstream" + ); + // After progress finished + state persisted, // we can forward messages directly to the downstream, // as backfill is finished. #[for_await] for msg in upstream { if let Some(msg) = mapping_message(msg?, &self.output_indices) { - tracing::trace!( - actor = self.actor_id, - message = ?msg, - "backfill_finished_after_barrier" - ); - if let Message::Barrier(barrier) = &msg { - self.state_table.commit_no_data_expected(barrier.epoch); - } yield msg; } } } + #[try_stream(ok = Option<(VirtualNode, OwnedRow)>, error = StreamExecutorError)] + async fn make_snapshot_stream( + upstream_table: &ReplicatedStateTable, + backfill_state: BackfillState, + paused: bool, + ) { + if paused { + #[for_await] + for _ in tokio_stream::pending() { + yield None; + } + } else { + #[for_await] + for r in Self::snapshot_read_per_vnode(upstream_table, backfill_state) { + yield r?; + } + } + } + + fn handle_snapshot_chunk( + chunk: DataChunk, + vnode: VirtualNode, + pk_in_output_indices: &[usize], + backfill_state: &mut BackfillState, + cur_barrier_snapshot_processed_rows: &mut u64, + total_snapshot_processed_rows: &mut u64, + output_indices: &[usize], + ) -> StreamExecutorResult { + let chunk = StreamChunk::from_parts(vec![Op::Insert; chunk.capacity()], chunk); + // Raise the current position. + // As snapshot read streams are ordered by pk, so we can + // just use the last row to update `current_pos`. + let snapshot_row_count_delta = chunk.cardinality() as u64; + update_pos_by_vnode( + vnode, + &chunk, + pk_in_output_indices, + backfill_state, + snapshot_row_count_delta, + )?; + + let chunk_cardinality = chunk.cardinality() as u64; + *cur_barrier_snapshot_processed_rows += chunk_cardinality; + *total_snapshot_processed_rows += chunk_cardinality; + let chunk = Message::Chunk(mapping_chunk(chunk, output_indices)); + Ok(chunk) + } + /// Read snapshot per vnode. /// These streams should be sorted in storage layer. /// 1. Get row iterator / vnode. @@ -527,22 +646,20 @@ where /// remaining data in `builder` must be flushed manually. /// Otherwise when we scan a new snapshot, it is possible the rows in the `builder` would be /// present, Then when we flush we contain duplicate rows. - #[try_stream(ok = Option<(VirtualNode, StreamChunk)>, error = StreamExecutorError)] - async fn snapshot_read_per_vnode<'a>( - upstream_table: &'a ReplicatedStateTable, + #[try_stream(ok = Option<(VirtualNode, OwnedRow)>, error = StreamExecutorError)] + async fn snapshot_read_per_vnode( + upstream_table: &ReplicatedStateTable, backfill_state: BackfillState, - builders: &'a mut [DataChunkBuilder], ) { - for (vnode, builder) in upstream_table - .vnodes() - .iter_vnodes() - .zip_eq_debug(builders.iter_mut()) - { + let mut iterators = vec![]; + for vnode in upstream_table.vnodes().iter_vnodes() { let backfill_progress = backfill_state.get_progress(&vnode)?; let current_pos = match backfill_progress { BackfillProgressPerVnode::NotStarted => None, - BackfillProgressPerVnode::Completed(current_pos) - | BackfillProgressPerVnode::InProgress(current_pos) => Some(current_pos.clone()), + BackfillProgressPerVnode::Completed { current_pos, .. } + | BackfillProgressPerVnode::InProgress { current_pos, .. } => { + Some(current_pos.clone()) + } }; let range_bounds = compute_bounds(upstream_table.pk_indices(), current_pos.clone()); @@ -558,26 +675,35 @@ where "iter_with_vnode_and_output_indices" ); let vnode_row_iter = upstream_table - .iter_with_vnode_and_output_indices(vnode, &range_bounds, Default::default()) + .iter_with_vnode_and_output_indices( + vnode, + &range_bounds, + PrefetchOptions::prefetch_for_small_range_scan(), + ) .await?; let vnode_row_iter = Box::pin(owned_row_iter(vnode_row_iter)); - let vnode_chunk_iter = - iter_chunks(vnode_row_iter, builder).map_ok(move |chunk| (vnode, chunk)); + let vnode_row_iter = vnode_row_iter.map_ok(move |row| (vnode, row)); - // This means we iterate serially rather than in parallel across vnodes. - #[for_await] - for chunk in vnode_chunk_iter { - yield Some(chunk?); - } + let vnode_row_iter = Box::pin(vnode_row_iter); + + iterators.push(vnode_row_iter); + } + + // TODO(kwannoel): We can provide an option between snapshot read in parallel vs serial. + let vnode_row_iter = select_all(iterators); + + #[for_await] + for vnode_and_row in vnode_row_iter { + yield Some(vnode_and_row?); } yield None; return Ok(()); } } -impl Executor for ArrangementBackfillExecutor +impl Execute for ArrangementBackfillExecutor where S: StateStore, SD: ValueRowSerde, @@ -585,16 +711,4 @@ where fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } diff --git a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs index 4c365fa78ec8c..d91bc2cf78cee 100644 --- a/src/stream/src/executor/backfill/cdc/cdc_backfill.rs +++ b/src/stream/src/executor/backfill/cdc/cdc_backfill.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::pin::{pin, Pin}; +use std::pin::Pin; use std::sync::Arc; use either::Either; @@ -26,12 +26,13 @@ use risingwave_common::row::{OwnedRow, Row, RowExt}; use risingwave_common::types::{DataType, ScalarRefImpl}; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_connector::parser::{ - DebeziumParser, EncodingProperties, JsonProperties, ProtocolProperties, + DebeziumParser, DebeziumProps, EncodingProperties, JsonProperties, ProtocolProperties, SourceStreamChunkBuilder, SpecificParserConfig, }; use risingwave_connector::source::cdc::external::CdcOffset; use risingwave_connector::source::{SourceColumnDesc, SourceContext}; use risingwave_storage::StateStore; +use rw_futures_util::pausable; use crate::common::table::state_table::StateTable; use crate::executor::backfill::cdc::state::CdcBackfillState; @@ -44,8 +45,8 @@ use crate::executor::backfill::utils::{ }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, StreamExecutorResult, }; use crate::task::CreateMviewProgress; @@ -54,13 +55,12 @@ const METADATA_STATE_LEN: usize = 4; pub struct CdcBackfillExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// The external table to be backfilled external_table: ExternalStorageTable, /// Upstream changelog stream which may contain metadata columns, e.g. `_rw_offset` - upstream: BoxedExecutor, + upstream: Executor, /// The column indices need to be forwarded to the downstream from the upstream and table scan. /// User may select a subset of columns from the upstream table. @@ -74,24 +74,25 @@ pub struct CdcBackfillExecutor { metrics: Arc, chunk_size: usize, + + disable_backfill: bool, } impl CdcBackfillExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, external_table: ExternalStorageTable, - upstream: BoxedExecutor, + upstream: Executor, output_indices: Vec, progress: Option, metrics: Arc, state_table: StateTable, chunk_size: usize, + disable_backfill: bool, ) -> Self { Self { actor_ctx, - info, external_table, upstream, output_indices, @@ -99,6 +100,7 @@ impl CdcBackfillExecutor { progress, metrics, chunk_size, + disable_backfill, } } @@ -122,6 +124,8 @@ impl CdcBackfillExecutor { // Poll the upstream to get the first barrier. let first_barrier = expect_first_barrier(&mut upstream).await?; + let mut paused = first_barrier.is_pause_on_startup(); + // Check whether this parallelism has been assigned splits, // if not, we should bypass the backfill directly. let mut state_impl = CdcBackfillState::new( @@ -140,7 +144,7 @@ impl CdcBackfillExecutor { let state = state_impl.restore_state().await?; current_pk_pos = state.current_pk_pos.clone(); - let to_backfill = !state.is_finished; + let to_backfill = !self.disable_backfill && !state.is_finished; // The first barrier message should be propagated. yield Message::Barrier(first_barrier); @@ -153,6 +157,10 @@ impl CdcBackfillExecutor { .last_cdc_offset .map_or(upstream_table_reader.current_binlog_offset().await?, Some); + let offset_parse_func = upstream_table_reader + .inner() + .table_reader() + .get_cdc_offset_parser(); let mut consumed_binlog_offset: Option = None; tracing::info!( @@ -161,6 +169,7 @@ impl CdcBackfillExecutor { initial_binlog_offset = ?last_binlog_offset, ?current_pk_pos, is_finished = state.is_finished, + disable_backfill = self.disable_backfill, snapshot_row_count = total_snapshot_row_count, chunk_size = self.chunk_size, "start cdc backfill" @@ -201,10 +210,7 @@ impl CdcBackfillExecutor { break; } Message::Chunk(ref chunk) => { - last_binlog_offset = get_cdc_chunk_last_offset( - upstream_table_reader.inner().table_reader(), - chunk, - )?; + last_binlog_offset = get_cdc_chunk_last_offset(&offset_parse_func, chunk)?; } Message::Watermark(_) => { // Ignore watermark @@ -219,8 +225,13 @@ impl CdcBackfillExecutor { let left_upstream = upstream.by_ref().map(Either::Left); let args = SnapshotReadArgs::new_for_cdc(current_pk_pos.clone(), self.chunk_size); - let right_snapshot = - pin!(upstream_table_reader.snapshot_read(args).map(Either::Right)); + + let (right_snapshot, valve) = + pausable(upstream_table_reader.snapshot_read(args).map(Either::Right)); + + if paused { + valve.pause(); + } // Prefer to select upstream, so we can stop snapshot stream when barrier comes. let backfill_stream = @@ -238,6 +249,21 @@ impl CdcBackfillExecutor { Either::Left(msg) => { match msg? { Message::Barrier(barrier) => { + if let Some(mutation) = barrier.mutation.as_deref() { + use crate::executor::Mutation; + match mutation { + Mutation::Pause => { + paused = true; + valve.pause(); + } + Mutation::Resume => { + paused = false; + valve.resume(); + } + _ => (), + } + } + // If it is a barrier, switch snapshot and consume buffered // upstream chunk. // If no current_pos, means we did not process any snapshot yet. @@ -250,12 +276,12 @@ impl CdcBackfillExecutor { // record the consumed binlog offset that will be // persisted later consumed_binlog_offset = get_cdc_chunk_last_offset( - upstream_table_reader.inner().table_reader(), + &offset_parse_func, &chunk, )?; yield Message::Chunk(mapping_chunk( mark_cdc_chunk( - upstream_table_reader.inner().table_reader(), + &offset_parse_func, chunk, current_pos, &pk_in_output_indices, @@ -318,10 +344,8 @@ impl CdcBackfillExecutor { continue; } - let chunk_binlog_offset = get_cdc_chunk_last_offset( - upstream_table_reader.inner().table_reader(), - &chunk, - )?; + let chunk_binlog_offset = + get_cdc_chunk_last_offset(&offset_parse_func, &chunk)?; tracing::trace!( target: "events::stream::cdc_backfill", @@ -411,6 +435,21 @@ impl CdcBackfillExecutor { } } } + } else if self.disable_backfill { + // If backfill is disabled, we just mark the backfill as finished + tracing::info!( + upstream_table_id, + upstream_table_name, + "CdcBackfill has been disabled" + ); + state_impl + .mutate_state( + current_pk_pos, + last_binlog_offset.clone(), + total_snapshot_row_count, + true, + ) + .await?; } // drop reader to release db connection @@ -418,6 +457,7 @@ impl CdcBackfillExecutor { tracing::info!( upstream_table_id, + upstream_table_name, "CdcBackfill has already finished and will forward messages directly to the downstream" ); @@ -467,7 +507,8 @@ pub async fn transform_upstream(upstream: BoxedMessageStream, schema: &Schema) { encoding_config: EncodingProperties::Json(JsonProperties { use_schema_registry: false, }), - protocol_config: ProtocolProperties::Debezium, + // the cdc message is generated internally so the key must exist. + protocol_config: ProtocolProperties::Debezium(DebeziumProps::default()), }; let mut parser = DebeziumParser::new( props, @@ -566,22 +607,10 @@ fn get_rw_columns(schema: &Schema) -> Vec { .collect_vec() } -impl Executor for CdcBackfillExecutor { +impl Execute for CdcBackfillExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } #[cfg(test)] @@ -596,7 +625,6 @@ mod tests { use crate::executor::backfill::cdc::cdc_backfill::transform_upstream; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; #[tokio::test] async fn test_transform_upstream_chunk() { @@ -606,7 +634,8 @@ mod tests { Field::unnamed(DataType::Varchar), // _rw_table_name ]); let pk_indices = vec![1]; - let (mut tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema.clone(), pk_indices.clone()); // let payload = r#"{"before": null,"after":{"O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" },"source":{"version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null},"op":"r","ts_ms":1695277757017,"transaction":null}"#.to_string(); let payload = r#"{ "payload": { "before": null, "after": { "O_ORDERKEY": 5, "O_CUSTKEY": 44485, "O_ORDERSTATUS": "F", "O_TOTALPRICE": "144659.20", "O_ORDERDATE": "1994-07-30" }, "source": { "version": "1.9.7.Final", "connector": "mysql", "name": "RW_CDC_1002", "ts_ms": 1695277757000, "snapshot": "last", "db": "mydb", "sequence": null, "table": "orders_new", "server_id": 0, "gtid": null, "file": "binlog.000008", "pos": 3693, "row": 0, "thread": null, "query": null }, "op": "r", "ts_ms": 1695277757017, "transaction": null } }"#; diff --git a/src/stream/src/executor/backfill/no_shuffle_backfill.rs b/src/stream/src/executor/backfill/no_shuffle_backfill.rs index 71b005235c40f..d9ee88b2d2264 100644 --- a/src/stream/src/executor/backfill/no_shuffle_backfill.rs +++ b/src/stream/src/executor/backfill/no_shuffle_backfill.rs @@ -17,14 +17,12 @@ use std::sync::Arc; use either::Either; use futures::stream::select_with_strategy; -use futures::{pin_mut, stream, StreamExt}; +use futures::{stream, StreamExt}; use futures_async_stream::try_stream; -use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; +use risingwave_common::array::{DataChunk, Op, StreamChunk}; use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::Datum; -use risingwave_common::util::chunk_coalesce::DataChunkBuilder; use risingwave_common::util::epoch::EpochPair; use risingwave_common::{bail, row}; use risingwave_hummock_sdk::HummockReadEpoch; @@ -35,13 +33,13 @@ use risingwave_storage::StateStore; use crate::common::table::state_table::StateTable; use crate::executor::backfill::utils; use crate::executor::backfill::utils::{ - compute_bounds, construct_initial_finished_state, create_builder, get_new_pos, iter_chunks, - mapping_chunk, mapping_message, mark_chunk, owned_row_iter, METADATA_STATE_LEN, + compute_bounds, construct_initial_finished_state, create_builder, get_new_pos, mapping_chunk, + mapping_message, mark_chunk, owned_row_iter, update_backfill_metrics, METADATA_STATE_LEN, }; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, Barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, - Message, Mutation, PkIndicesRef, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, Barrier, BoxedMessageStream, Execute, Executor, Message, Mutation, + StreamExecutorError, StreamExecutorResult, }; use crate::task::{ActorId, CreateMviewProgress}; @@ -77,12 +75,10 @@ pub struct BackfillState { /// in the same worker, so that we can read uncommitted data from the upstream table without /// waiting. pub struct BackfillExecutor { - info: ExecutorInfo, - /// Upstream table upstream_table: StorageTable, /// Upstream with the same schema with the upstream table. - upstream: BoxedExecutor, + upstream: Executor, /// Internal state table for persisting state of backfill state. state_table: Option>, @@ -111,9 +107,8 @@ where { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, upstream_table: StorageTable, - upstream: BoxedExecutor, + upstream: Executor, state_table: Option>, output_indices: Vec, progress: CreateMviewProgress, @@ -123,7 +118,6 @@ where ) -> Self { let actor_id = progress.actor_id(); Self { - info, upstream_table, upstream, state_table, @@ -138,12 +132,15 @@ where #[try_stream(ok = Message, error = StreamExecutorError)] async fn execute_inner(mut self) { - // The primary key columns, in the output columns of the upstream_table scan. - let pk_in_output_indices = self.upstream_table.pk_in_output_indices().unwrap(); + // The primary key columns. + // We receive a pruned chunk from the upstream table, + // which will only contain output columns of the scan on the upstream table. + // The pk indices specify the pk columns of the pruned chunk. + let pk_indices = self.upstream_table.pk_in_output_indices().unwrap(); let mut rate_limit = self.rate_limit; - let state_len = pk_in_output_indices.len() + METADATA_STATE_LEN; + let state_len = pk_indices.len() + METADATA_STATE_LEN; let pk_order = self.upstream_table.pk_serializer().get_order_types(); @@ -153,6 +150,7 @@ where // Poll the upstream to get the first barrier. let first_barrier = expect_first_barrier(&mut upstream).await?; + let mut paused = first_barrier.is_pause_on_startup(); let init_epoch = first_barrier.epoch.prev; if let Some(state_table) = self.state_table.as_mut() { state_table.init_epoch(first_barrier.epoch); @@ -163,15 +161,11 @@ where is_finished, row_count, mut old_state, - } = Self::recover_backfill_state(self.state_table.as_ref(), pk_in_output_indices.len()) - .await?; + } = Self::recover_backfill_state(self.state_table.as_ref(), pk_indices.len()).await?; tracing::trace!(is_finished, row_count, "backfill state recovered"); - let mut builder = create_builder( - rate_limit, - self.chunk_size, - self.upstream_table.schema().data_types(), - ); + let data_types = self.upstream_table.schema().data_types(); + let mut builder = create_builder(rate_limit, self.chunk_size, data_types.clone()); // Use this buffer to construct state, // which will then be persisted. @@ -234,14 +228,13 @@ where { let left_upstream = upstream.by_ref().map(Either::Left); - let right_snapshot = pin!(Self::snapshot_read( + let right_snapshot = pin!(Self::make_snapshot_stream( &self.upstream_table, snapshot_read_epoch, current_pos.clone(), - true, - &mut builder + paused ) - .map(Either::Right),); + .map(Either::Right)); // Prefer to select upstream, so we can stop snapshot stream as soon as the // barrier comes. @@ -281,6 +274,18 @@ where has_snapshot_read = true; match msg? { None => { + // Consume remaining rows in the builder. + if let Some(data_chunk) = builder.consume_all() { + yield Self::handle_snapshot_chunk( + data_chunk, + &mut current_pos, + &mut cur_barrier_snapshot_processed_rows, + &mut total_snapshot_processed_rows, + &pk_indices, + &self.output_indices, + ); + } + // End of the snapshot read stream. // We should not mark the chunk anymore, // otherwise, we will ignore some rows @@ -296,23 +301,27 @@ where &self.output_indices, )); } - + update_backfill_metrics( + &self.metrics, + self.actor_id, + upstream_table_id, + cur_barrier_snapshot_processed_rows, + cur_barrier_upstream_processed_rows, + ); break 'backfill_loop; } - Some(chunk) => { - // Raise the current position. - // As snapshot read streams are ordered by pk, so we can - // just use the last row to update `current_pos`. - current_pos = - Some(get_new_pos(&chunk, &pk_in_output_indices)); - - let chunk_cardinality = chunk.cardinality() as u64; - cur_barrier_snapshot_processed_rows += chunk_cardinality; - total_snapshot_processed_rows += chunk_cardinality; - yield Message::Chunk(mapping_chunk( - chunk, - &self.output_indices, - )); + Some(record) => { + // Buffer the snapshot read row. + if let Some(data_chunk) = builder.append_one_row(record) { + yield Self::handle_snapshot_chunk( + data_chunk, + &mut current_pos, + &mut cur_barrier_snapshot_processed_rows, + &mut total_snapshot_processed_rows, + &pk_indices, + &self.output_indices, + ); + } } } } @@ -322,7 +331,12 @@ where // Before processing barrier, if did not snapshot read, // do a snapshot read first. // This is so we don't lose the tombstone iteration progress. - if !has_snapshot_read { + // If paused, we also can't read any snapshot records. + if !has_snapshot_read && !paused { + assert!( + builder.is_empty(), + "Builder should be empty if no snapshot read" + ); let (_, snapshot) = backfill_stream.into_inner(); #[for_await] for msg in snapshot { @@ -337,19 +351,16 @@ where snapshot_read_complete = true; break; } - Some(chunk) => { - // Raise the current position. - // As snapshot read streams are ordered by pk, so we can - // just use the last row to update `current_pos`. - current_pos = Some(get_new_pos(&chunk, &pk_in_output_indices)); - - let chunk_cardinality = chunk.cardinality() as u64; - cur_barrier_snapshot_processed_rows += chunk_cardinality; - total_snapshot_processed_rows += chunk_cardinality; - yield Message::Chunk(mapping_chunk( + Some(row) => { + let chunk = DataChunk::from_rows(&[row], &data_types); + yield Self::handle_snapshot_chunk( chunk, + &mut current_pos, + &mut cur_barrier_snapshot_processed_rows, + &mut total_snapshot_processed_rows, + &pk_indices, &self.output_indices, - )); + ); break; } } @@ -372,14 +383,14 @@ where // Consume snapshot rows left in builder let chunk = builder.consume_all(); if let Some(chunk) = chunk { - let chunk_cardinality = chunk.cardinality() as u64; - let ops = vec![Op::Insert; chunk.capacity()]; - let chunk = StreamChunk::from_parts(ops, chunk); - current_pos = Some(get_new_pos(&chunk, &pk_in_output_indices)); - - cur_barrier_snapshot_processed_rows += chunk_cardinality; - total_snapshot_processed_rows += chunk_cardinality; - yield Message::Chunk(mapping_chunk(chunk, &self.output_indices)); + yield Self::handle_snapshot_chunk( + chunk, + &mut current_pos, + &mut cur_barrier_snapshot_processed_rows, + &mut total_snapshot_processed_rows, + &pk_indices, + &self.output_indices, + ); } // Consume upstream buffer chunk @@ -390,7 +401,7 @@ where for chunk in upstream_chunk_buffer.drain(..) { cur_barrier_upstream_processed_rows += chunk.cardinality() as u64; yield Message::Chunk(mapping_chunk( - mark_chunk(chunk, current_pos, &pk_in_output_indices, pk_order), + mark_chunk(chunk, current_pos, &pk_indices, pk_order), &self.output_indices, )); } @@ -398,21 +409,13 @@ where upstream_chunk_buffer.clear() } - self.metrics - .backfill_snapshot_read_row_count - .with_label_values(&[ - upstream_table_id.to_string().as_str(), - self.actor_id.to_string().as_str(), - ]) - .inc_by(cur_barrier_snapshot_processed_rows); - - self.metrics - .backfill_upstream_output_row_count - .with_label_values(&[ - upstream_table_id.to_string().as_str(), - self.actor_id.to_string().as_str(), - ]) - .inc_by(cur_barrier_upstream_processed_rows); + update_backfill_metrics( + &self.metrics, + self.actor_id, + upstream_table_id, + cur_barrier_snapshot_processed_rows, + cur_barrier_upstream_processed_rows, + ); // Update snapshot read epoch. snapshot_read_epoch = barrier.epoch.prev; @@ -443,23 +446,32 @@ where ); // Update snapshot read chunk builder. - if let Some(mutation) = barrier.mutation.as_ref() { - if let Mutation::Throttle(actor_to_apply) = mutation.as_ref() { - let new_rate_limit_entry = actor_to_apply.get(&self.actor_id); - if let Some(new_rate_limit) = new_rate_limit_entry { - rate_limit = new_rate_limit.as_ref().map(|x| *x as _); - tracing::info!( - id = self.actor_id, - new_rate_limit = ?self.rate_limit, - "actor rate limit changed", - ); - assert!(builder.is_empty()); - builder = create_builder( - rate_limit, - self.chunk_size, - self.upstream_table.schema().data_types(), - ); + if let Some(mutation) = barrier.mutation.as_deref() { + match mutation { + Mutation::Pause => { + paused = true; } + Mutation::Resume => { + paused = false; + } + Mutation::Throttle(actor_to_apply) => { + let new_rate_limit_entry = actor_to_apply.get(&self.actor_id); + if let Some(new_rate_limit) = new_rate_limit_entry { + rate_limit = new_rate_limit.as_ref().map(|x| *x as _); + tracing::info!( + id = self.actor_id, + new_rate_limit = ?self.rate_limit, + "actor rate limit changed", + ); + assert!(builder.is_empty()); + builder = create_builder( + rate_limit, + self.chunk_size, + self.upstream_table.schema().data_types(), + ); + } + } + _ => (), } } @@ -491,8 +503,7 @@ where // since it expects to have been initialized in previous epoch // (there's no epoch before the first epoch). if current_pos.is_none() { - current_pos = - Some(construct_initial_finished_state(pk_in_output_indices.len())) + current_pos = Some(construct_initial_finished_state(pk_indices.len())) } // We will update current_pos at least once, @@ -530,6 +541,9 @@ where yield msg; break; } + // Allow other messages to pass through. + // We won't yield twice here, since if there's a barrier, + // we will always break out of the loop. yield msg; } } @@ -608,19 +622,37 @@ where } } + #[try_stream(ok = Option, error = StreamExecutorError)] + async fn make_snapshot_stream( + upstream_table: &StorageTable, + epoch: u64, + current_pos: Option, + paused: bool, + ) { + if paused { + #[for_await] + for _ in tokio_stream::pending() { + yield None; + } + } else { + #[for_await] + for r in Self::snapshot_read(upstream_table, epoch, current_pos) { + yield r?; + } + } + } + /// Snapshot read the upstream mv. /// The rows from upstream snapshot read will be buffered inside the `builder`. /// If snapshot is dropped before its rows are consumed, /// remaining data in `builder` must be flushed manually. /// Otherwise when we scan a new snapshot, it is possible the rows in the `builder` would be /// present, Then when we flush we contain duplicate rows. - #[try_stream(ok = Option, error = StreamExecutorError)] - async fn snapshot_read<'a>( - upstream_table: &'a StorageTable, + #[try_stream(ok = Option, error = StreamExecutorError)] + async fn snapshot_read( + upstream_table: &StorageTable, epoch: u64, current_pos: Option, - ordered: bool, - builder: &'a mut DataChunkBuilder, ) { let range_bounds = compute_bounds(upstream_table.pk_indices(), current_pos); let range_bounds = match range_bounds { @@ -638,17 +670,16 @@ where HummockReadEpoch::NoWait(epoch), row::empty(), range_bounds, - ordered, + true, // Here we only use small range prefetch because every barrier change, the executor will recreate a new iterator. So we do not need prefetch too much data. PrefetchOptions::prefetch_for_small_range_scan(), ) .await?; let row_iter = owned_row_iter(iter); - pin_mut!(row_iter); #[for_await] - for chunk in iter_chunks(row_iter, builder) { - yield Some(chunk?); + for row in row_iter { + yield Some(row?); } yield None; } @@ -675,25 +706,40 @@ where ) .await } + + /// 1. Converts from data chunk to stream chunk. + /// 2. Update the current position. + /// 3. Update Metrics + /// 4. Map the chunk according to output indices, return + /// the stream message to be yielded downstream. + fn handle_snapshot_chunk( + data_chunk: DataChunk, + current_pos: &mut Option, + cur_barrier_snapshot_processed_rows: &mut u64, + total_snapshot_processed_rows: &mut u64, + pk_indices: &[usize], + output_indices: &[usize], + ) -> Message { + let ops = vec![Op::Insert; data_chunk.capacity()]; + let chunk = StreamChunk::from_parts(ops, data_chunk); + // Raise the current position. + // As snapshot read streams are ordered by pk, so we can + // just use the last row to update `current_pos`. + *current_pos = Some(get_new_pos(&chunk, pk_indices)); + + let chunk_cardinality = chunk.cardinality() as u64; + *cur_barrier_snapshot_processed_rows += chunk_cardinality; + *total_snapshot_processed_rows += chunk_cardinality; + + Message::Chunk(mapping_chunk(chunk, output_indices)) + } } -impl Executor for BackfillExecutor +impl Execute for BackfillExecutor where S: StateStore, { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } diff --git a/src/stream/src/executor/backfill/utils.rs b/src/stream/src/executor/backfill/utils.rs index 1a19a3fe201fc..f7b28e2adce3a 100644 --- a/src/stream/src/executor/backfill/utils.rs +++ b/src/stream/src/executor/backfill/utils.rs @@ -34,16 +34,16 @@ use risingwave_common::util::iter_util::ZipEqDebug; use risingwave_common::util::sort_util::{cmp_datum_iter, OrderType}; use risingwave_common::util::value_encoding::BasicSerde; use risingwave_connector::error::ConnectorError; -use risingwave_connector::source::cdc::external::{ - CdcOffset, ExternalTableReader, ExternalTableReaderImpl, -}; +use risingwave_connector::source::cdc::external::{CdcOffset, CdcOffsetParseFunc}; use risingwave_storage::table::{collect_data_chunk_with_builder, KeyedRow}; use risingwave_storage::StateStore; use crate::common::table::state_table::StateTableInner; +use crate::executor::monitor::StreamingMetrics; use crate::executor::{ Message, PkIndicesRef, StreamExecutorError, StreamExecutorResult, Watermark, }; +use crate::task::ActorId; /// `vnode`, `is_finished`, `row_count`, all occupy 1 column each. pub const METADATA_STATE_LEN: usize = 3; @@ -90,12 +90,24 @@ impl BackfillState { &mut self, vnode: VirtualNode, new_pos: OwnedRow, + snapshot_row_count_delta: u64, ) -> StreamExecutorResult<()> { let state = self.get_current_state(&vnode); - let new_state = BackfillProgressPerVnode::InProgress(new_pos); match state { - BackfillProgressPerVnode::NotStarted => *state = new_state, - BackfillProgressPerVnode::InProgress(_current_pos) => *state = new_state, + BackfillProgressPerVnode::NotStarted => { + *state = BackfillProgressPerVnode::InProgress { + current_pos: new_pos, + snapshot_row_count: snapshot_row_count_delta, + }; + } + BackfillProgressPerVnode::InProgress { + snapshot_row_count, .. + } => { + *state = BackfillProgressPerVnode::InProgress { + current_pos: new_pos, + snapshot_row_count: *snapshot_row_count + snapshot_row_count_delta, + }; + } BackfillProgressPerVnode::Completed { .. } => unreachable!(), } Ok(()) @@ -104,14 +116,20 @@ impl BackfillState { pub(crate) fn finish_progress(&mut self, vnode: VirtualNode, pos_len: usize) { let finished_placeholder_position = construct_initial_finished_state(pos_len); let current_state = self.get_current_state(&vnode); - let new_pos = match current_state { - BackfillProgressPerVnode::NotStarted => finished_placeholder_position, - BackfillProgressPerVnode::InProgress(current_pos) => current_pos.clone(), + let (new_pos, snapshot_row_count) = match current_state { + BackfillProgressPerVnode::NotStarted => (finished_placeholder_position, 0), + BackfillProgressPerVnode::InProgress { + current_pos, + snapshot_row_count, + } => (current_pos.clone(), *snapshot_row_count), BackfillProgressPerVnode::Completed { .. } => { return; } }; - *current_state = BackfillProgressPerVnode::Completed(new_pos); + *current_state = BackfillProgressPerVnode::Completed { + current_pos: new_pos, + snapshot_row_count, + }; } /// Return state to be committed. @@ -119,42 +137,56 @@ impl BackfillState { let new_state = self.inner.get(vnode).unwrap().current_state().clone(); let new_encoded_state = match new_state { BackfillProgressPerVnode::NotStarted => unreachable!(), - BackfillProgressPerVnode::InProgress(current_pos) => { + BackfillProgressPerVnode::InProgress { + current_pos, + snapshot_row_count, + } => { let mut encoded_state = vec![None; current_pos.len() + METADATA_STATE_LEN]; encoded_state[0] = Some(vnode.to_scalar().into()); encoded_state[1..current_pos.len() + 1].clone_from_slice(current_pos.as_inner()); encoded_state[current_pos.len() + 1] = Some(false.into()); - encoded_state[current_pos.len() + 2] = Some(0i64.into()); + encoded_state[current_pos.len() + 2] = Some((snapshot_row_count as i64).into()); encoded_state } - BackfillProgressPerVnode::Completed(current_pos) => { + BackfillProgressPerVnode::Completed { + current_pos, + snapshot_row_count, + } => { let mut encoded_state = vec![None; current_pos.len() + METADATA_STATE_LEN]; encoded_state[0] = Some(vnode.to_scalar().into()); encoded_state[1..current_pos.len() + 1].clone_from_slice(current_pos.as_inner()); encoded_state[current_pos.len() + 1] = Some(true.into()); - encoded_state[current_pos.len() + 2] = Some(0i64.into()); + encoded_state[current_pos.len() + 2] = Some((snapshot_row_count as i64).into()); encoded_state } }; let old_state = self.inner.get(vnode).unwrap().committed_state().clone(); let old_encoded_state = match old_state { BackfillProgressPerVnode::NotStarted => None, - BackfillProgressPerVnode::InProgress(committed_pos) => { + BackfillProgressPerVnode::InProgress { + current_pos, + snapshot_row_count, + } => { + let committed_pos = current_pos; let mut encoded_state = vec![None; committed_pos.len() + METADATA_STATE_LEN]; encoded_state[0] = Some(vnode.to_scalar().into()); encoded_state[1..committed_pos.len() + 1] .clone_from_slice(committed_pos.as_inner()); encoded_state[committed_pos.len() + 1] = Some(false.into()); - encoded_state[committed_pos.len() + 2] = Some(0i64.into()); + encoded_state[committed_pos.len() + 2] = Some((snapshot_row_count as i64).into()); Some(encoded_state) } - BackfillProgressPerVnode::Completed(committed_pos) => { + BackfillProgressPerVnode::Completed { + current_pos, + snapshot_row_count, + } => { + let committed_pos = current_pos; let mut encoded_state = vec![None; committed_pos.len() + METADATA_STATE_LEN]; encoded_state[0] = Some(vnode.to_scalar().into()); encoded_state[1..committed_pos.len() + 1] .clone_from_slice(committed_pos.as_inner()); encoded_state[committed_pos.len() + 1] = Some(true.into()); - encoded_state[committed_pos.len() + 2] = Some(0i64.into()); + encoded_state[committed_pos.len() + 2] = Some((snapshot_row_count as i64).into()); Some(encoded_state) } }; @@ -167,8 +199,8 @@ impl BackfillState { let state = self.inner.get(vnode).unwrap(); match state.current_state() { // If current state and committed state are the same, we don't need to commit. - s @ BackfillProgressPerVnode::InProgress(_current_pos) - | s @ BackfillProgressPerVnode::Completed(_current_pos) => s != state.committed_state(), + s @ BackfillProgressPerVnode::InProgress { .. } + | s @ BackfillProgressPerVnode::Completed { .. } => s != state.committed_state(), BackfillProgressPerVnode::NotStarted => false, } } @@ -181,10 +213,18 @@ impl BackfillState { assert!(matches!( current_state, - BackfillProgressPerVnode::InProgress(_) | BackfillProgressPerVnode::Completed(_) + BackfillProgressPerVnode::InProgress { .. } + | BackfillProgressPerVnode::Completed { .. } )); *committed_state = current_state.clone(); } + + pub(crate) fn get_snapshot_row_count(&self) -> u64 { + self.inner + .values() + .map(|p| p.get_snapshot_row_count()) + .sum() + } } #[derive(Clone, Debug, PartialEq, Eq)] @@ -211,6 +251,10 @@ impl BackfillStatePerVnode { pub(crate) fn current_state(&self) -> &BackfillProgressPerVnode { &self.current_state } + + pub(crate) fn get_snapshot_row_count(&self) -> u64 { + self.current_state().get_snapshot_row_count() + } } impl From> for BackfillState { @@ -227,8 +271,32 @@ impl From> for BackfillState { pub enum BackfillProgressPerVnode { /// no entry exists for a vnode, or on initialization of the executor. NotStarted, - InProgress(OwnedRow), - Completed(OwnedRow), + InProgress { + /// The current snapshot offset + current_pos: OwnedRow, + /// Number of snapshot records read for this vnode. + snapshot_row_count: u64, + }, + Completed { + /// The current snapshot offset + current_pos: OwnedRow, + /// Number of snapshot records read for this vnode. + snapshot_row_count: u64, + }, +} + +impl BackfillProgressPerVnode { + fn get_snapshot_row_count(&self) -> u64 { + match self { + BackfillProgressPerVnode::NotStarted => 0, + BackfillProgressPerVnode::InProgress { + snapshot_row_count, .. + } + | BackfillProgressPerVnode::Completed { + snapshot_row_count, .. + } => *snapshot_row_count, + } + } } pub(crate) fn mark_chunk( @@ -242,7 +310,7 @@ pub(crate) fn mark_chunk( } pub(crate) fn mark_cdc_chunk( - table_reader: &ExternalTableReaderImpl, + offset_parse_func: &CdcOffsetParseFunc, chunk: StreamChunk, current_pos: &OwnedRow, pk_in_output_indices: PkIndicesRef<'_>, @@ -251,7 +319,7 @@ pub(crate) fn mark_cdc_chunk( ) -> StreamExecutorResult { let chunk = chunk.compact(); mark_cdc_chunk_inner( - table_reader, + offset_parse_func, chunk, current_pos, last_cdc_offset, @@ -275,16 +343,14 @@ pub(crate) fn mark_chunk_ref_by_vnode( let mut new_visibility = BitmapBuilder::with_capacity(ops.len()); // Use project to avoid allocation. for row in data.rows() { - // TODO(kwannoel): Is this logic correct for computing vnode? - // I will revisit it again when arrangement_backfill is implemented e2e. let vnode = VirtualNode::compute_row(row, pk_in_output_indices); let v = match backfill_state.get_progress(&vnode)? { // We want to just forward the row, if the vnode has finished backfill. - BackfillProgressPerVnode::Completed(_) => true, + BackfillProgressPerVnode::Completed { .. } => true, // If not started, no need to forward. BackfillProgressPerVnode::NotStarted => false, // If in progress, we need to check row <= current_pos. - BackfillProgressPerVnode::InProgress(current_pos) => { + BackfillProgressPerVnode::InProgress { current_pos, .. } => { let lhs = row.project(pk_in_output_indices); let rhs = current_pos; let order = cmp_datum_iter(lhs.iter(), rhs.iter(), pk_order.iter().copied()); @@ -332,7 +398,7 @@ fn mark_chunk_inner( } fn mark_cdc_chunk_inner( - table_reader: &ExternalTableReaderImpl, + offset_parse_func: &CdcOffsetParseFunc, chunk: StreamChunk, current_pos: &OwnedRow, last_cdc_offset: Option, @@ -346,7 +412,7 @@ fn mark_cdc_chunk_inner( let offset_col_idx = data.dimension() - 1; for v in data.rows().map(|row| { let offset_datum = row.datum_at(offset_col_idx).unwrap(); - let event_offset = table_reader.parse_cdc_offset(offset_datum.into_utf8())?; + let event_offset = (*offset_parse_func)(offset_datum.into_utf8())?; let visible = { // filter changelog events with binlog range let in_binlog_range = if let Some(binlog_low) = &last_cdc_offset { @@ -402,45 +468,71 @@ pub(crate) fn mapping_message(msg: Message, upstream_indices: &[usize]) -> Optio } /// Recovers progress per vnode, so we know which to backfill. +/// See how it decodes the state with the inline comments. pub(crate) async fn get_progress_per_vnode( state_table: &StateTableInner, ) -> StreamExecutorResult> { debug_assert!(!state_table.vnodes().is_empty()); let vnodes = state_table.vnodes().iter_vnodes(); let mut result = Vec::with_capacity(state_table.vnodes().len()); + // 1. Get the vnode keys, so we can get the state per vnode. let vnode_keys = vnodes.map(|vnode| { let datum: [Datum; 1] = [Some(vnode.to_scalar().into())]; datum }); let tasks = vnode_keys.map(|vnode_key| state_table.get_row(vnode_key)); - let states_for_vnode_keys = try_join_all(tasks).await?; - for (vnode, state_for_vnode_key) in state_table + // 2. Fetch the state for each vnode. + // It should have the following schema, it should not contain vnode: + // | pk | `backfill_finished` | `row_count` | + let state_for_vnodes = try_join_all(tasks).await?; + for (vnode, state_for_vnode) in state_table .vnodes() .iter_vnodes() - .zip_eq_debug(states_for_vnode_keys) + .zip_eq_debug(state_for_vnodes) { - // NOTE(kwannoel): state_for_vnode_key does not include the vnode prefix. - let backfill_progress = match state_for_vnode_key { + let backfill_progress = match state_for_vnode { + // There's some state, means there was progress made. It's either finished / in progress. Some(row) => { + // 3. Decode the `snapshot_row_count`. Decode from the back, since + // pk is variable length. + let snapshot_row_count = row.as_inner().get(row.len() - 1).unwrap(); + let snapshot_row_count = (*snapshot_row_count.as_ref().unwrap().as_int64()) as u64; + + // 4. Decode the `is_finished` flag (whether backfill has finished). + // Decode from the back, since pk is variable length. let vnode_is_finished = row.as_inner().get(row.len() - 2).unwrap(); let vnode_is_finished = vnode_is_finished.as_ref().unwrap(); - // Only the current pos should be contained in the in-memory backfill state. - // Row count will be added later. + // 5. Decode the `current_pos`. let current_pos = row.as_inner().get(..row.len() - 2).unwrap(); let current_pos = current_pos.into_owned_row(); + + // 6. Construct the in-memory state per vnode, based on the decoded state. if *vnode_is_finished.as_bool() { BackfillStatePerVnode::new( - BackfillProgressPerVnode::Completed(current_pos.clone()), - BackfillProgressPerVnode::Completed(current_pos), + BackfillProgressPerVnode::Completed { + current_pos: current_pos.clone(), + snapshot_row_count, + }, + BackfillProgressPerVnode::Completed { + current_pos, + snapshot_row_count, + }, ) } else { BackfillStatePerVnode::new( - BackfillProgressPerVnode::InProgress(current_pos.clone()), - BackfillProgressPerVnode::InProgress(current_pos), + BackfillProgressPerVnode::InProgress { + current_pos: current_pos.clone(), + snapshot_row_count, + }, + BackfillProgressPerVnode::InProgress { + current_pos, + snapshot_row_count, + }, ) } } + // No state, means no progress made. None => BackfillStatePerVnode::new( BackfillProgressPerVnode::NotStarted, BackfillProgressPerVnode::NotStarted, @@ -461,10 +553,7 @@ pub(crate) async fn flush_data( ) -> StreamExecutorResult<()> { let vnodes = table.vnodes().clone(); if let Some(old_state) = old_state { - if old_state[1..] == current_partial_state[1..] { - table.commit_no_data_expected(epoch); - return Ok(()); - } else { + if old_state[1..] != current_partial_state[1..] { vnodes.iter_vnodes_scalar().for_each(|vnode| { let datum = Some(vnode.into()); current_partial_state[0] = datum.clone(); @@ -512,10 +601,11 @@ pub(crate) fn update_pos_by_vnode( chunk: &StreamChunk, pk_in_output_indices: &[usize], backfill_state: &mut BackfillState, + snapshot_row_count_delta: u64, ) -> StreamExecutorResult<()> { let new_pos = get_new_pos(chunk, pk_in_output_indices); assert_eq!(new_pos.len(), pk_in_output_indices.len()); - backfill_state.update_progress(vnode, new_pos)?; + backfill_state.update_progress(vnode, new_pos, snapshot_row_count_delta)?; Ok(()) } @@ -532,13 +622,13 @@ pub(crate) fn get_new_pos(chunk: &StreamChunk, pk_in_output_indices: &[usize]) - } pub(crate) fn get_cdc_chunk_last_offset( - table_reader: &ExternalTableReaderImpl, + offset_parse_func: &CdcOffsetParseFunc, chunk: &StreamChunk, ) -> StreamExecutorResult> { let row = chunk.rows().last().unwrap().1; let offset_col = row.iter().last().unwrap(); - let output = offset_col - .map(|scalar| Ok::<_, ConnectorError>(table_reader.parse_cdc_offset(scalar.into_utf8()))?); + let output = + offset_col.map(|scalar| Ok::<_, ConnectorError>((*offset_parse_func)(scalar.into_utf8()))?); output.transpose().map_err(|e| e.into()) } @@ -604,7 +694,7 @@ where } /// Schema -/// | vnode | pk | `backfill_finished` | +/// | vnode | pk | `backfill_finished` | `row_count` | /// Persists the state per vnode based on `BackfillState`. /// We track the current committed state via `committed_progress` /// so we know whether we need to persist the state or not. @@ -614,12 +704,12 @@ where /// - Not persist to store at all. /// /// `InProgress`: -/// - Format: | vnode | pk | false | +/// - Format: | vnode | pk | false | `row_count` | /// - If change in current pos: Persist. /// - No change in current pos: Do not persist. /// /// Completed -/// - Format: | vnode | pk | true | +/// - Format: | vnode | pk | true | `row_count` | /// - If previous state is `InProgress` / `NotStarted`: Persist. /// - If previous state is Completed: Do not persist. /// TODO(kwannoel): we should check committed state to be all `finished` in the tests. @@ -632,7 +722,6 @@ pub(crate) async fn persist_state_per_vnode, ) -> StreamExecutorResult<()> { - let mut has_progress = false; for vnode in vnodes { if !backfill_state.need_commit(&vnode) { continue; @@ -667,7 +756,6 @@ pub(crate) async fn persist_state_per_vnode( flush_data(table, epoch, old_state, current_state).await?; *old_state = Some(current_state.into()); } else { - table.commit_no_data_expected(epoch); + table.commit(epoch).await?; } Ok(()) } @@ -733,3 +817,27 @@ pub fn create_builder( DataChunkBuilder::new(data_types, chunk_size) } } + +pub fn update_backfill_metrics( + metrics: &StreamingMetrics, + actor_id: ActorId, + upstream_table_id: u32, + cur_barrier_snapshot_processed_rows: u64, + cur_barrier_upstream_processed_rows: u64, +) { + metrics + .backfill_snapshot_read_row_count + .with_label_values(&[ + upstream_table_id.to_string().as_str(), + actor_id.to_string().as_str(), + ]) + .inc_by(cur_barrier_snapshot_processed_rows); + + metrics + .backfill_upstream_output_row_count + .with_label_values(&[ + upstream_table_id.to_string().as_str(), + actor_id.to_string().as_str(), + ]) + .inc_by(cur_barrier_upstream_processed_rows); +} diff --git a/src/stream/src/executor/barrier_recv.rs b/src/stream/src/executor/barrier_recv.rs index 391dbefb1dc54..79f6157ab6bcd 100644 --- a/src/stream/src/executor/barrier_recv.rs +++ b/src/stream/src/executor/barrier_recv.rs @@ -13,52 +13,37 @@ // limitations under the License. use futures::StreamExt; -use risingwave_common::catalog::Schema; use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ - ActorContext, ActorContextRef, Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, - PkIndices, PkIndicesRef, StreamExecutorError, + ActorContext, ActorContextRef, Barrier, BoxedMessageStream, Execute, Message, + StreamExecutorError, }; /// The executor only for receiving barrier from the meta service. It always resides in the leaves /// of the streaming graph. pub struct BarrierRecvExecutor { _ctx: ActorContextRef, - info: ExecutorInfo, /// The barrier receiver registered in the local barrier manager. barrier_receiver: UnboundedReceiver, } impl BarrierRecvExecutor { - pub fn new( - ctx: ActorContextRef, - info: ExecutorInfo, - barrier_receiver: UnboundedReceiver, - ) -> Self { + pub fn new(ctx: ActorContextRef, barrier_receiver: UnboundedReceiver) -> Self { Self { _ctx: ctx, - info, barrier_receiver, } } pub fn for_test(barrier_receiver: UnboundedReceiver) -> Self { - Self::new( - ActorContext::create(0), - ExecutorInfo { - schema: Schema::empty().clone(), - pk_indices: PkIndices::new(), - identity: "BarrierRecvExecutor".to_string(), - }, - barrier_receiver, - ) + Self::new(ActorContext::for_test(0), barrier_receiver) } } -impl Executor for BarrierRecvExecutor { +impl Execute for BarrierRecvExecutor { fn execute(self: Box) -> BoxedMessageStream { UnboundedReceiverStream::new(self.barrier_receiver) .map(|barrier| Ok(Message::Barrier(barrier))) @@ -69,18 +54,6 @@ impl Executor for BarrierRecvExecutor { })) .boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } #[cfg(test)] diff --git a/src/stream/src/executor/batch_query.rs b/src/stream/src/executor/batch_query.rs index 533a327432f43..7c92bcd732423 100644 --- a/src/stream/src/executor/batch_query.rs +++ b/src/stream/src/executor/batch_query.rs @@ -24,7 +24,7 @@ use risingwave_storage::table::collect_data_chunk; use risingwave_storage::StateStore; use super::error::StreamExecutorError; -use super::{Executor, ExecutorInfo, Message}; +use super::{Execute, Message}; use crate::executor::BoxedMessageStream; pub struct BatchQueryExecutor { @@ -34,18 +34,18 @@ pub struct BatchQueryExecutor { /// The number of tuples in one [`StreamChunk`] batch_size: usize, - info: ExecutorInfo, + schema: Schema, } impl BatchQueryExecutor where S: StateStore, { - pub fn new(table: StorageTable, batch_size: usize, info: ExecutorInfo) -> Self { + pub fn new(table: StorageTable, batch_size: usize, schema: Schema) -> Self { Self { table, batch_size, - info, + schema, } } @@ -62,7 +62,7 @@ where pin_mut!(iter); while let Some(data_chunk) = - collect_data_chunk(&mut iter, self.schema(), Some(self.batch_size)) + collect_data_chunk(&mut iter, &self.schema, Some(self.batch_size)) .instrument_await("batch_query_executor_collect_chunk") .await? { @@ -73,7 +73,7 @@ where } } -impl Executor for BatchQueryExecutor +impl Execute for BatchQueryExecutor where S: StateStore, { @@ -81,18 +81,6 @@ where unreachable!("should call `execute_with_epoch`") } - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - fn execute_with_epoch(self: Box, epoch: u64) -> BoxedMessageStream { self.execute_inner(epoch).boxed() } @@ -100,9 +88,6 @@ where #[cfg(test)] mod test { - - use std::vec; - use futures_async_stream::for_await; use super::*; @@ -114,15 +99,10 @@ mod test { let test_batch_count = 5; let table = gen_basic_table(test_batch_count * test_batch_size).await; - let info = ExecutorInfo { - schema: table.schema().clone(), - pk_indices: vec![0, 1], - identity: "BatchQuery".to_owned(), - }; - - let executor = Box::new(BatchQueryExecutor::new(table, test_batch_size, info)); - - let stream = executor.execute_with_epoch(u64::MAX); + let schema = table.schema().clone(); + let stream = BatchQueryExecutor::new(table, test_batch_size, schema) + .boxed() + .execute_with_epoch(u64::MAX); let mut batch_cnt = 0; #[for_await] diff --git a/src/stream/src/executor/chain.rs b/src/stream/src/executor/chain.rs index 676c4bde46dd1..c8b7b25852e74 100644 --- a/src/stream/src/executor/chain.rs +++ b/src/stream/src/executor/chain.rs @@ -14,10 +14,9 @@ use futures::StreamExt; use futures_async_stream::try_stream; -use risingwave_common::catalog::Schema; use super::error::StreamExecutorError; -use super::{expect_first_barrier, BoxedExecutor, Executor, ExecutorInfo, Message}; +use super::{expect_first_barrier, Execute, Executor, Message}; use crate::task::{ActorId, CreateMviewProgress}; /// [`ChainExecutor`] is an executor that enables synchronization between the existing stream and @@ -25,11 +24,9 @@ use crate::task::{ActorId, CreateMviewProgress}; /// feature. It pipes new data of existing MVs to newly created MV only all of the old data in the /// existing MVs are dispatched. pub struct ChainExecutor { - info: ExecutorInfo, + snapshot: Executor, - snapshot: BoxedExecutor, - - upstream: BoxedExecutor, + upstream: Executor, progress: CreateMviewProgress, @@ -41,14 +38,12 @@ pub struct ChainExecutor { impl ChainExecutor { pub fn new( - info: ExecutorInfo, - snapshot: BoxedExecutor, - upstream: BoxedExecutor, + snapshot: Executor, + upstream: Executor, progress: CreateMviewProgress, upstream_only: bool, ) -> Self { Self { - info, snapshot, upstream, actor_id: progress.actor_id(), @@ -104,22 +99,10 @@ impl ChainExecutor { } } -impl Executor for ChainExecutor { +impl Execute for ChainExecutor { fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } #[cfg(test)] @@ -136,9 +119,7 @@ mod test { use super::ChainExecutor; use crate::executor::test_utils::MockSource; - use crate::executor::{ - AddMutation, Barrier, Executor, ExecutorInfo, Message, Mutation, PkIndices, - }; + use crate::executor::{AddMutation, Barrier, Execute, Message, Mutation, PkIndices}; use crate::task::{CreateMviewProgress, LocalBarrierManager}; #[tokio::test] @@ -148,48 +129,35 @@ mod test { let actor_id = progress.actor_id(); let schema = Schema::new(vec![Field::unnamed(DataType::Int64)]); - let first = Box::new( - MockSource::with_chunks( - schema.clone(), - PkIndices::new(), - vec![ - StreamChunk::from_pretty("I\n + 1"), - StreamChunk::from_pretty("I\n + 2"), - ], - ) - .stop_on_finish(false), - ); - - let second = Box::new(MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1)).with_mutation( - Mutation::Add(AddMutation { - adds: maplit::hashmap! { - 0 => vec![Dispatcher { - downstream_actor_id: vec![actor_id], - ..Default::default() - }], - }, - added_actors: maplit::hashset! { actor_id }, - splits: Default::default(), - pause: false, - }), - )), - Message::Chunk(StreamChunk::from_pretty("I\n + 3")), - Message::Chunk(StreamChunk::from_pretty("I\n + 4")), - ], - )); - - let info = ExecutorInfo { - schema, - pk_indices: PkIndices::new(), - identity: "ChainExecutor".to_string(), - }; - let chain = ChainExecutor::new(info, first, second, progress, false); - - let mut chain = Box::new(chain).execute(); + let first = MockSource::with_chunks(vec![ + StreamChunk::from_pretty("I\n + 1"), + StreamChunk::from_pretty("I\n + 2"), + ]) + .stop_on_finish(false) + .into_executor(schema.clone(), PkIndices::new()); + + let second = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1)).with_mutation( + Mutation::Add(AddMutation { + adds: maplit::hashmap! { + 0 => vec![Dispatcher { + downstream_actor_id: vec![actor_id], + ..Default::default() + }], + }, + added_actors: maplit::hashset! { actor_id }, + splits: Default::default(), + pause: false, + }), + )), + Message::Chunk(StreamChunk::from_pretty("I\n + 3")), + Message::Chunk(StreamChunk::from_pretty("I\n + 4")), + ]) + .into_executor(schema.clone(), PkIndices::new()); + + let chain = ChainExecutor::new(first, second, progress, false); + + let mut chain = chain.boxed().execute(); chain.next().await; let mut count = 0; diff --git a/src/stream/src/executor/dedup/append_only_dedup.rs b/src/stream/src/executor/dedup/append_only_dedup.rs index b731b9088c25e..7b171a1ac844d 100644 --- a/src/stream/src/executor/dedup/append_only_dedup.rs +++ b/src/stream/src/executor/dedup/append_only_dedup.rs @@ -19,7 +19,6 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::BitmapBuilder; -use risingwave_common::catalog::Schema; use risingwave_common::row::{OwnedRow, Row, RowExt}; use risingwave_storage::StateStore; @@ -29,39 +28,39 @@ use crate::common::table::state_table::StateTable; use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorResult, }; use crate::task::AtomicU64Ref; /// [`AppendOnlyDedupExecutor`] drops any message that has duplicate pk columns with previous /// messages. It only accepts append-only input, and its output will be append-only as well. pub struct AppendOnlyDedupExecutor { - input: Option, + ctx: ActorContextRef, + + input: Option, + dedup_cols: Vec, state_table: StateTable, cache: DedupCache, - - info: ExecutorInfo, - ctx: ActorContextRef, } impl AppendOnlyDedupExecutor { pub fn new( - input: BoxedExecutor, - state_table: StateTable, - info: ExecutorInfo, ctx: ActorContextRef, + input: Executor, + dedup_cols: Vec, + state_table: StateTable, watermark_epoch: AtomicU64Ref, metrics: Arc, ) -> Self { let metrics_info = MetricsInfo::new(metrics, state_table.table_id(), ctx.id, "AppendOnly Dedup"); Self { + ctx, input: Some(input), + dedup_cols, state_table, cache: DedupCache::new(watermark_epoch, metrics_info), - info, - ctx, } } @@ -76,8 +75,6 @@ impl AppendOnlyDedupExecutor { // The first barrier message should be propagated. yield Message::Barrier(barrier); - let mut commit_data = false; - #[for_await] for msg in input { self.cache.evict(); @@ -92,7 +89,7 @@ impl AppendOnlyDedupExecutor { .data_chunk() .rows_with_holes() .map(|row_ref| { - row_ref.map(|row| row.project(self.pk_indices()).to_owned_row()) + row_ref.map(|row| row.project(&self.dedup_cols).to_owned_row()) }) .collect_vec(); @@ -127,21 +124,13 @@ impl AppendOnlyDedupExecutor { let chunk = StreamChunk::with_visibility(ops, columns, vis); self.state_table.write_chunk(chunk.clone()); - commit_data = true; - yield Message::Chunk(chunk); } self.state_table.try_flush().await?; } Message::Barrier(barrier) => { - if commit_data { - // Only commit when we have new data in this epoch. - self.state_table.commit(barrier.epoch).await?; - commit_data = false; - } else { - self.state_table.commit_no_data_expected(barrier.epoch); - } + self.state_table.commit(barrier.epoch).await?; if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(self.ctx.id) { let (_prev_vnode_bitmap, cache_may_stale) = @@ -195,22 +184,10 @@ impl AppendOnlyDedupExecutor { } } -impl Executor for AppendOnlyDedupExecutor { +impl Execute for AppendOnlyDedupExecutor { fn execute(self: Box) -> BoxedMessageStream { self.executor_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } #[cfg(test)] @@ -241,7 +218,8 @@ mod tests { Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), ]); - let pk_indices = vec![0]; + let dedup_col_indices = vec![0]; + let pk_indices = dedup_col_indices.clone(); let order_types = vec![OrderType::ascending()]; let state_store = MemoryStateStore::new(); @@ -254,20 +232,17 @@ mod tests { ) .await; - let (mut tx, input) = MockSource::channel(schema.clone(), pk_indices.clone()); - let info = ExecutorInfo { - schema, - pk_indices, - identity: "AppendOnlyDedupExecutor".to_string(), - }; - let mut dedup_executor = Box::new(AppendOnlyDedupExecutor::new( - Box::new(input), + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, pk_indices); + let mut dedup_executor = AppendOnlyDedupExecutor::new( + ActorContext::for_test(123), + source, + dedup_col_indices, state_table, - info, - ActorContext::create(123), Arc::new(AtomicU64::new(0)), Arc::new(StreamingMetrics::unused()), - )) + ) + .boxed() .execute(); tx.push_barrier(test_epoch(1), false); diff --git a/src/stream/src/executor/dispatch.rs b/src/stream/src/executor/dispatch.rs index 1dc7385f0f3c0..a2b229d118ed9 100644 --- a/src/stream/src/executor/dispatch.rs +++ b/src/stream/src/executor/dispatch.rs @@ -35,17 +35,17 @@ use tokio::time::Instant; use tracing::{event, Instrument}; use super::exchange::output::{new_output, BoxedOutput}; -use super::{AddMutation, UpdateMutation, Watermark}; +use super::{AddMutation, Executor, UpdateMutation, Watermark}; use crate::error::StreamResult; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{Barrier, BoxedExecutor, Message, Mutation, StreamConsumer}; +use crate::executor::{Barrier, Message, Mutation, StreamConsumer}; use crate::task::{ActorId, DispatcherId, SharedContext}; /// [`DispatchExecutor`] consumes messages and send them into downstream actors. Usually, /// data chunks will be dispatched with some specified policy, while control message /// such as barriers will be distributed to all receivers. pub struct DispatchExecutor { - input: BoxedExecutor, + input: Executor, inner: DispatchExecutorInner, } @@ -341,7 +341,7 @@ impl DispatchExecutorInner { impl DispatchExecutor { pub fn new( - input: BoxedExecutor, + input: Executor, dispatchers: Vec, actor_id: u32, fragment_id: u32, @@ -386,16 +386,22 @@ impl StreamConsumer for DispatchExecutor { #[for_await] for msg in input { let msg: Message = msg?; - let (barrier, span) = match msg { - Message::Chunk(_) => (None, "dispatch_chunk"), - Message::Barrier(ref barrier) => (Some(barrier.clone()), "dispatch_barrier"), - Message::Watermark(_) => (None, "dispatch_watermark"), - }; - - let tracing_span = if let Some(_barrier) = &barrier { - tracing::info_span!("dispatch_barrier") - } else { - tracing::Span::none() + let (barrier, span, tracing_span) = match msg { + Message::Chunk(_) => ( + None, + "dispatch_chunk", + tracing::info_span!("dispatch_chunk"), + ), + Message::Barrier(ref barrier) => ( + Some(barrier.clone()), + "dispatch_barrier", + tracing::info_span!("dispatch_barrier"), + ), + Message::Watermark(_) => ( + None, + "dispatch_watermark", + tracing::info_span!("dispatch_watermark"), + ), }; self.inner @@ -619,7 +625,14 @@ impl RoundRobinDataDispatcher { impl Dispatcher for RoundRobinDataDispatcher { async fn dispatch_data(&mut self, chunk: StreamChunk) -> StreamResult<()> { - let chunk = chunk.project(&self.output_indices); + let chunk = if self.output_indices.len() < chunk.columns().len() { + chunk + .project(&self.output_indices) + .eliminate_adjacent_noop_update() + } else { + chunk.project(&self.output_indices) + }; + self.outputs[self.cur].send(Message::Chunk(chunk)).await?; self.cur += 1; self.cur %= self.outputs.len(); @@ -739,7 +752,13 @@ impl Dispatcher for HashDataDispatcher { let mut new_ops: Vec = Vec::with_capacity(chunk.capacity()); // Apply output indices after calculating the vnode. - let chunk = chunk.project(&self.output_indices); + let chunk = if self.output_indices.len() < chunk.columns().len() { + chunk + .project(&self.output_indices) + .eliminate_adjacent_noop_update() + } else { + chunk.project(&self.output_indices) + }; for ((vnode, &op), visible) in vnodes .iter() @@ -858,7 +877,13 @@ impl BroadcastDispatcher { impl Dispatcher for BroadcastDispatcher { async fn dispatch_data(&mut self, chunk: StreamChunk) -> StreamResult<()> { - let chunk = chunk.project(&self.output_indices); + let chunk = if self.output_indices.len() < chunk.columns().len() { + chunk + .project(&self.output_indices) + .eliminate_adjacent_noop_update() + } else { + chunk.project(&self.output_indices) + }; broadcast_concurrent(self.outputs.values_mut(), Message::Chunk(chunk)).await } @@ -956,7 +981,13 @@ impl Dispatcher for SimpleDispatcher { .exactly_one() .expect("expect exactly one output"); - let chunk = chunk.project(&self.output_indices); + let chunk = if self.output_indices.len() < chunk.columns().len() { + chunk + .project(&self.output_indices) + .eliminate_adjacent_noop_update() + } else { + chunk.project(&self.output_indices) + }; output.send(Message::Chunk(chunk)).await } @@ -1012,6 +1043,7 @@ mod tests { use crate::executor::exchange::output::Output; use crate::executor::exchange::permit::channel_for_test; use crate::executor::receiver::ReceiverExecutor; + use crate::executor::Execute; use crate::task::test_utils::helper_make_local_actor; #[derive(Debug)] @@ -1122,7 +1154,7 @@ mod tests { let (tx, rx) = channel_for_test(); let actor_id = 233; let fragment_id = 666; - let input = Box::new(ReceiverExecutor::for_test(rx)); + let input = Executor::new(Default::default(), ReceiverExecutor::for_test(rx).boxed()); let ctx = Arc::new(SharedContext::for_test()); let metrics = Arc::new(StreamingMetrics::unused()); @@ -1179,7 +1211,7 @@ mod tests { // 2. Take downstream receivers. let mut rxs = [untouched, old, new, old_simple, new_simple] .into_iter() - .map(|id| (id, ctx.take_receiver(&(actor_id, id)).unwrap())) + .map(|id| (id, ctx.take_receiver((actor_id, id)).unwrap())) .collect::>(); macro_rules! try_recv { ($down_id:expr) => { diff --git a/src/stream/src/executor/dml.rs b/src/stream/src/executor/dml.rs index 03b67e6495f89..69230ae60735b 100644 --- a/src/stream/src/executor/dml.rs +++ b/src/stream/src/executor/dml.rs @@ -16,28 +16,23 @@ use std::collections::BTreeMap; use std::mem; use either::Either; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; -use risingwave_common::catalog::{ColumnDesc, Schema, TableId, TableVersionId}; +use risingwave_common::catalog::{ColumnDesc, TableId, TableVersionId}; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::transaction::transaction_message::TxnMsg; -use risingwave_source::dml_manager::DmlManagerRef; +use risingwave_dml::dml_manager::DmlManagerRef; use super::error::StreamExecutorError; -use super::{ - expect_first_barrier, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, - Mutation, PkIndicesRef, -}; +use super::{expect_first_barrier, BoxedMessageStream, Execute, Executor, Message, Mutation}; use crate::common::StreamChunkBuilder; use crate::executor::stream_reader::StreamReaderWithPause; /// [`DmlExecutor`] accepts both stream data and batch data for data manipulation on a specific /// table. The two streams will be merged into one and then sent to downstream. pub struct DmlExecutor { - info: ExecutorInfo, - - upstream: BoxedExecutor, + upstream: Executor, /// Stores the information of batch data channels. dml_manager: DmlManagerRef, @@ -71,10 +66,8 @@ struct TxnBuffer { } impl DmlExecutor { - #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, - upstream: BoxedExecutor, + upstream: Executor, dml_manager: DmlManagerRef, table_id: TableId, table_version_id: TableVersionId, @@ -82,7 +75,6 @@ impl DmlExecutor { chunk_size: usize, ) -> Self { Self { - info, upstream, dml_manager, table_id, @@ -106,16 +98,21 @@ impl DmlExecutor { // Note(bugen): Only register after the first barrier message is received, which means the // current executor is activated. This avoids the new reader overwriting the old one during // the preparation of schema change. - let batch_reader = self - .dml_manager - .register_reader(self.table_id, self.table_version_id, &self.column_descs) - .map_err(StreamExecutorError::connector_error)?; - let batch_reader = batch_reader.stream_reader().into_stream(); + let handle = self.dml_manager.register_reader( + self.table_id, + self.table_version_id, + &self.column_descs, + )?; + let reader = handle + .stream_reader() + .into_stream() + .map_err(StreamExecutorError::from) + .boxed(); // Merge the two streams using `StreamReaderWithPause` because when we receive a pause // barrier, we should stop receiving the data from DML. We poll data from the two streams in // a round robin way. - let mut stream = StreamReaderWithPause::::new(upstream, batch_reader); + let mut stream = StreamReaderWithPause::::new(upstream, reader); // If the first barrier requires us to pause on startup, pause the stream. if barrier.is_pause_on_startup() { @@ -272,22 +269,10 @@ impl DmlExecutor { } } -impl Executor for DmlExecutor { +impl Execute for DmlExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } #[cfg(test)] @@ -295,12 +280,12 @@ mod tests { use std::sync::Arc; use risingwave_common::array::StreamChunk; - use risingwave_common::catalog::{ColumnId, Field, INITIAL_TABLE_VERSION_ID}; + use risingwave_common::catalog::{ColumnId, Field, Schema, INITIAL_TABLE_VERSION_ID}; use risingwave_common::test_prelude::StreamChunkTestExt; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::types::DataType; use risingwave_common::util::epoch::test_epoch; - use risingwave_source::dml_manager::DmlManager; + use risingwave_dml::dml_manager::DmlManager; use super::*; use crate::executor::test_utils::MockSource; @@ -322,23 +307,18 @@ mod tests { let pk_indices = vec![0]; let dml_manager = Arc::new(DmlManager::for_test()); - let (mut tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); - let info = ExecutorInfo { - schema, - pk_indices, - identity: "DmlExecutor".to_string(), - }; + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, pk_indices); - let dml_executor = Box::new(DmlExecutor::new( - info, - Box::new(source), + let dml_executor = DmlExecutor::new( + source, dml_manager.clone(), table_id, INITIAL_TABLE_VERSION_ID, column_descs, 1024, - )); - let mut dml_executor = dml_executor.execute(); + ); + let mut dml_executor = dml_executor.boxed().execute(); let stream_chunk1 = StreamChunk::from_pretty( " I I diff --git a/src/stream/src/executor/dynamic_filter.rs b/src/stream/src/executor/dynamic_filter.rs index a82741510312a..eb5a95991051c 100644 --- a/src/stream/src/executor/dynamic_filter.rs +++ b/src/stream/src/executor/dynamic_filter.rs @@ -38,10 +38,7 @@ use risingwave_storage::StateStore; use super::barrier_align::*; use super::error::StreamExecutorError; use super::monitor::StreamingMetrics; -use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, - PkIndicesRef, -}; +use super::{ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message}; use crate::common::table::state_table::{StateTable, WatermarkCacheParameterizedStateTable}; use crate::common::StreamChunkBuilder; use crate::executor::expect_first_barrier_from_aligned_stream; @@ -49,10 +46,12 @@ use crate::task::ActorEvalErrorReport; pub struct DynamicFilterExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - source_l: Option, - source_r: Option, + eval_error_report: ActorEvalErrorReport, + + schema: Schema, + source_l: Option, + source_r: Option, key_l: usize, comparator: ExprNodeType, left_table: WatermarkCacheParameterizedStateTable, @@ -71,9 +70,9 @@ impl DynamicFilterExecutor, @@ -83,9 +82,14 @@ impl DynamicFilterExecutor Self { + let eval_error_report = ActorEvalErrorReport { + actor_context: ctx.clone(), + identity: Arc::from(info.identity.as_str()), + }; Self { ctx, - info, + eval_error_report, + schema: info.schema.clone(), source_l: Some(source_l), source_r: Some(source_r), key_l, @@ -263,7 +267,7 @@ impl DynamicFilterExecutor DynamicFilterExecutor DynamicFilterExecutor DynamicFilterExecutor DynamicFilterExecutor Executor +impl Execute for DynamicFilterExecutor { fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity + self.execute_inner().boxed() } } @@ -566,21 +551,20 @@ mod tests { let schema = Schema { fields: vec![Field::unnamed(DataType::Int64)], }; - let (tx_l, source_l) = MockSource::channel(schema.clone(), vec![0]); - let (tx_r, source_r) = MockSource::channel(schema, vec![]); - - let schema = source_l.schema().clone(); - let info = ExecutorInfo { - schema, - pk_indices: vec![0], - identity: "DynamicFilterExecutor".to_string(), - }; + let (tx_l, source_l) = MockSource::channel(); + let source_l = source_l.into_executor(schema.clone(), vec![0]); + let (tx_r, source_r) = MockSource::channel(); + let source_r = source_r.into_executor(schema, vec![]); let executor = DynamicFilterExecutor::::new( - ActorContext::create(123), - info, - Box::new(source_l), - Box::new(source_r), + ActorContext::for_test(123), + &ExecutorInfo { + schema: source_l.schema().clone(), + pk_indices: vec![0], + identity: "DynamicFilterExecutor".to_string(), + }, + source_l, + source_r, 0, comparator, mem_state_l, @@ -590,7 +574,7 @@ mod tests { always_relax, false, ); - (tx_l, tx_r, Box::new(executor).execute()) + (tx_l, tx_r, executor.boxed().execute()) } #[tokio::test] diff --git a/src/stream/src/executor/error.rs b/src/stream/src/executor/error.rs index 0a00aa9f0b678..8b3e8bf5212b2 100644 --- a/src/stream/src/executor/error.rs +++ b/src/stream/src/executor/error.rs @@ -17,6 +17,7 @@ use risingwave_common::error::{BoxedError, NotImplemented}; use risingwave_common::util::value_encoding::error::ValueEncodingError; use risingwave_connector::error::ConnectorError; use risingwave_connector::sink::SinkError; +use risingwave_dml::error::DmlError; use risingwave_expr::ExprError; use risingwave_pb::PbFieldNotFound; use risingwave_rpc_client::error::RpcError; @@ -87,11 +88,11 @@ pub enum ErrorKind { BoxedError, ), - #[error("Dml error: {0}")] + #[error(transparent)] DmlError( - #[source] + #[from] #[backtrace] - BoxedError, + DmlError, ), #[error(transparent)] diff --git a/src/stream/src/executor/exchange/input.rs b/src/stream/src/executor/exchange/input.rs index 42d2c5da95f1a..5eb583a1caddd 100644 --- a/src/stream/src/executor/exchange/input.rs +++ b/src/stream/src/executor/exchange/input.rs @@ -236,7 +236,7 @@ pub(crate) fn new_input( let input = if is_local_address(&context.addr, &upstream_addr) { LocalInput::new( - context.take_receiver(&(upstream_actor_id, actor_id))?, + context.take_receiver((upstream_actor_id, actor_id))?, upstream_actor_id, ) .boxed_input() diff --git a/src/stream/src/executor/expand.rs b/src/stream/src/executor/expand.rs index cb8c45732d17f..2375ebca3d0df 100644 --- a/src/stream/src/executor/expand.rs +++ b/src/stream/src/executor/expand.rs @@ -16,26 +16,19 @@ use std::fmt::Debug; use futures::StreamExt; use futures_async_stream::try_stream; -use risingwave_common::array::{Array, I64Array}; -use risingwave_common::catalog::Schema; +use risingwave_common::array::{Array, I64Array, StreamChunk}; use super::error::StreamExecutorError; -use super::*; +use super::{BoxedMessageStream, Execute, Executor, Message}; pub struct ExpandExecutor { - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, column_subsets: Vec>, } impl ExpandExecutor { - pub fn new( - info: ExecutorInfo, - input: Box, - column_subsets: Vec>, - ) -> Self { + pub fn new(input: Executor, column_subsets: Vec>) -> Self { Self { - info, input, column_subsets, } @@ -73,19 +66,7 @@ impl Debug for ExpandExecutor { } } -impl Executor for ExpandExecutor { - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - +impl Execute for ExpandExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } @@ -100,7 +81,7 @@ mod tests { use super::ExpandExecutor; use crate::executor::test_utils::MockSource; - use crate::executor::{Executor, ExecutorInfo, PkIndices}; + use crate::executor::{Execute, PkIndices}; #[tokio::test] async fn test_expand() { @@ -111,29 +92,19 @@ mod tests { + 6 6 3 - 7 5 4", ); - let input_schema = Schema { - fields: vec![ + let source = MockSource::with_chunks(vec![chunk1]).into_executor( + Schema::new(vec![ Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), Field::unnamed(DataType::Int64), - ], - }; - let source = MockSource::with_chunks(input_schema.clone(), PkIndices::new(), vec![chunk1]); - let schema = { - let mut fields = input_schema.into_fields(); - fields.extend(fields.clone()); - fields.push(Field::with_name(DataType::Int64, "flag")); - Schema::new(fields) - }; - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "ExpandExecutor".to_string(), - }; + ]), + PkIndices::new(), + ); let column_subsets = vec![vec![0, 1], vec![1, 2]]; - let expand = Box::new(ExpandExecutor::new(info, Box::new(source), column_subsets)); - let mut expand = expand.execute(); + let mut expand = ExpandExecutor::new(source, column_subsets) + .boxed() + .execute(); let chunk = expand.next().await.unwrap().unwrap().into_chunk().unwrap(); assert_eq!( diff --git a/src/stream/src/executor/filter.rs b/src/stream/src/executor/filter.rs index 34ec98367cc7f..ffc8847c42abf 100644 --- a/src/stream/src/executor/filter.rs +++ b/src/stream/src/executor/filter.rs @@ -15,13 +15,17 @@ use std::fmt::{Debug, Formatter}; use std::sync::Arc; +use futures::StreamExt; +use futures_async_stream::try_stream; use risingwave_common::array::{Array, ArrayImpl, Op, StreamChunk}; use risingwave_common::buffer::BitmapBuilder; -use risingwave_common::catalog::Schema; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; -use super::*; +use super::{ + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, StreamExecutorError, + StreamExecutorResult, +}; /// `FilterExecutor` filters data with the `expr`. The `expr` takes a chunk of data, /// and returns a boolean array on whether each item should be retained. And then, @@ -29,8 +33,7 @@ use super::*; /// to the result of the expression. pub struct FilterExecutor { _ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, /// Expression of the current filter, note that the filter must always have the same output for /// the same input. @@ -38,15 +41,9 @@ pub struct FilterExecutor { } impl FilterExecutor { - pub fn new( - ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, - expr: NonStrictExpression, - ) -> Self { + pub fn new(ctx: ActorContextRef, input: Executor, expr: NonStrictExpression) -> Self { Self { _ctx: ctx, - info, input, expr, } @@ -135,19 +132,7 @@ impl Debug for FilterExecutor { } } -impl Executor for FilterExecutor { - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - +impl Execute for FilterExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } @@ -218,23 +203,14 @@ mod tests { ], }; let pk_indices = PkIndices::new(); - let source = - MockSource::with_chunks(schema.clone(), pk_indices.clone(), vec![chunk1, chunk2]); - let info = ExecutorInfo { - schema, - pk_indices, - identity: "FilterExecutor".to_string(), - }; + let source = MockSource::with_chunks(vec![chunk1, chunk2]) + .into_executor(schema.clone(), pk_indices.clone()); let test_expr = build_from_pretty("(greater_than:boolean $0:int8 $1:int8)"); - let filter = Box::new(FilterExecutor::new( - ActorContext::create(123), - info, - Box::new(source), - test_expr, - )); - let mut filter = filter.execute(); + let mut filter = FilterExecutor::new(ActorContext::for_test(123), source, test_expr) + .boxed() + .execute(); let chunk = filter.next().await.unwrap().unwrap().into_chunk().unwrap(); assert_eq!( diff --git a/src/stream/src/executor/flow_control.rs b/src/stream/src/executor/flow_control.rs index 53d0f15a602ae..4e4212a1e246e 100644 --- a/src/stream/src/executor/flow_control.rs +++ b/src/stream/src/executor/flow_control.rs @@ -15,11 +15,14 @@ use std::fmt::{Debug, Formatter}; use std::num::NonZeroU32; +use futures::StreamExt; +use futures_async_stream::try_stream; use governor::clock::MonotonicClock; use governor::{Quota, RateLimiter}; -use risingwave_common::catalog::Schema; -use super::*; +use super::{ + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, Mutation, StreamExecutorError, +}; /// Flow Control Executor is used to control the rate of the input executor. /// @@ -31,27 +34,16 @@ use super::*; /// /// It is used to throttle problematic MVs that are consuming too much resources. pub struct FlowControlExecutor { - input: BoxedExecutor, + input: Executor, actor_ctx: ActorContextRef, - identity: String, rate_limit: Option, } impl FlowControlExecutor { - pub fn new( - input: Box, - actor_ctx: ActorContextRef, - rate_limit: Option, - ) -> Self { - let identity = if rate_limit.is_some() { - format!("{} (flow controlled)", input.identity()) - } else { - input.identity().to_owned() - }; + pub fn new(input: Executor, actor_ctx: ActorContextRef, rate_limit: Option) -> Self { Self { input, actor_ctx, - identity, rate_limit, } } @@ -128,20 +120,8 @@ impl Debug for FlowControlExecutor { } } -impl Executor for FlowControlExecutor { +impl Execute for FlowControlExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - self.input.schema() - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - self.input.pk_indices() - } - - fn identity(&self) -> &str { - &self.identity - } } diff --git a/src/stream/src/executor/hash_agg.rs b/src/stream/src/executor/hash_agg.rs index 5078fd0bc5154..7e3e47a665600 100644 --- a/src/stream/src/executor/hash_agg.rs +++ b/src/stream/src/executor/hash_agg.rs @@ -39,8 +39,7 @@ use super::aggregation::{ }; use super::sort_buffer::SortBuffer; use super::{ - expect_first_barrier, ActorContextRef, ExecutorInfo, PkIndicesRef, StreamExecutorResult, - Watermark, + expect_first_barrier, ActorContextRef, Executor, ExecutorInfo, StreamExecutorResult, Watermark, }; use crate::cache::{cache_may_stale, new_with_hasher, ManagedLruCache}; use crate::common::metrics::MetricsInfo; @@ -49,7 +48,7 @@ use crate::common::StreamChunkBuilder; use crate::error::StreamResult; use crate::executor::aggregation::AggGroup as GenericAggGroup; use crate::executor::error::StreamExecutorError; -use crate::executor::{BoxedMessageStream, Executor, Message}; +use crate::executor::{BoxedMessageStream, Execute, Message}; use crate::task::AtomicU64Ref; type AggGroup = GenericAggGroup; @@ -74,7 +73,7 @@ type AggGroupCache = ManagedLruCache>, Precompu /// all modifications will be flushed to the storage backend. Meanwhile, the executor will go /// through `group_change_set`, and produce a stream chunk based on the state changes. pub struct HashAggExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } @@ -193,27 +192,15 @@ impl ExecutionStats { } } -impl Executor for HashAggExecutor { +impl Execute for HashAggExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } } impl HashAggExecutor { pub fn new(args: AggExecutorArgs) -> StreamResult { - let input_info = args.input.info(); + let input_info = args.input.info().clone(); let group_key_len = args.extra.group_key_indices.len(); // NOTE: we assume the prefix of table pk is exactly the group key @@ -352,7 +339,7 @@ impl HashAggExecutor { chunk: StreamChunk, ) -> StreamExecutorResult<()> { // Find groups in this chunk and generate visibility for each group key. - let keys = K::build(&this.group_key_indices, chunk.data_chunk())?; + let keys = K::build_many(&this.group_key_indices, chunk.data_chunk()); let group_visibilities = Self::get_group_visibilities(keys, chunk.visibility()); // Ensure all `AggGroup`s are in `dirty_groups`. diff --git a/src/stream/src/executor/hash_join.rs b/src/stream/src/executor/hash_join.rs index 29852ffda3308..cb018d72f6e74 100644 --- a/src/stream/src/executor/hash_join.rs +++ b/src/stream/src/executor/hash_join.rs @@ -22,7 +22,6 @@ use futures_async_stream::try_stream; use itertools::Itertools; use multimap::MultiMap; use risingwave_common::array::{Op, RowRef, StreamChunk}; -use risingwave_common::catalog::Schema; use risingwave_common::hash::{HashKey, NullBitmap}; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{DataType, DefaultOrd, ToOwnedDatum}; @@ -33,112 +32,25 @@ use risingwave_expr::ExprError; use risingwave_storage::StateStore; use tokio::time::Instant; -use self::JoinType::{FullOuter, LeftOuter, LeftSemi, RightAnti, RightOuter, RightSemi}; +use self::builder::JoinChunkBuilder; use super::barrier_align::*; use super::error::{StreamExecutorError, StreamExecutorResult}; -use super::managed_state::join::*; +use super::join::hash_join::*; +use super::join::row::JoinRow; +use super::join::{JoinTypePrimitive, SideTypePrimitive, *}; use super::monitor::StreamingMetrics; use super::watermark::*; use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, Message, - PkIndicesRef, Watermark, + ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, Watermark, }; use crate::common::table::state_table::StateTable; -use crate::common::JoinStreamChunkBuilder; use crate::executor::expect_first_barrier_from_aligned_stream; -use crate::executor::JoinType::LeftAnti; +use crate::executor::join::builder::JoinStreamChunkBuilder; use crate::task::AtomicU64Ref; -/// The `JoinType` and `SideType` are to mimic a enum, because currently -/// enum is not supported in const generic. -// TODO: Use enum to replace this once [feature(adt_const_params)](https://github.com/rust-lang/rust/issues/95174) get completed. -pub type JoinTypePrimitive = u8; - /// Evict the cache every n rows. const EVICT_EVERY_N_ROWS: u32 = 16; -#[allow(non_snake_case, non_upper_case_globals)] -pub mod JoinType { - use super::JoinTypePrimitive; - pub const Inner: JoinTypePrimitive = 0; - pub const LeftOuter: JoinTypePrimitive = 1; - pub const RightOuter: JoinTypePrimitive = 2; - pub const FullOuter: JoinTypePrimitive = 3; - pub const LeftSemi: JoinTypePrimitive = 4; - pub const LeftAnti: JoinTypePrimitive = 5; - pub const RightSemi: JoinTypePrimitive = 6; - pub const RightAnti: JoinTypePrimitive = 7; -} - -pub type SideTypePrimitive = u8; -#[allow(non_snake_case, non_upper_case_globals)] -pub mod SideType { - use super::SideTypePrimitive; - pub const Left: SideTypePrimitive = 0; - pub const Right: SideTypePrimitive = 1; -} - -const fn is_outer_side(join_type: JoinTypePrimitive, side_type: SideTypePrimitive) -> bool { - join_type == JoinType::FullOuter - || (join_type == JoinType::LeftOuter && side_type == SideType::Left) - || (join_type == JoinType::RightOuter && side_type == SideType::Right) -} - -const fn outer_side_null(join_type: JoinTypePrimitive, side_type: SideTypePrimitive) -> bool { - join_type == JoinType::FullOuter - || (join_type == JoinType::LeftOuter && side_type == SideType::Right) - || (join_type == JoinType::RightOuter && side_type == SideType::Left) -} - -/// Send the update only once if the join type is semi/anti and the update is the same side as the -/// join -const fn forward_exactly_once(join_type: JoinTypePrimitive, side_type: SideTypePrimitive) -> bool { - ((join_type == JoinType::LeftSemi || join_type == JoinType::LeftAnti) - && side_type == SideType::Left) - || ((join_type == JoinType::RightSemi || join_type == JoinType::RightAnti) - && side_type == SideType::Right) -} - -const fn only_forward_matched_side( - join_type: JoinTypePrimitive, - side_type: SideTypePrimitive, -) -> bool { - ((join_type == JoinType::LeftSemi || join_type == JoinType::LeftAnti) - && side_type == SideType::Right) - || ((join_type == JoinType::RightSemi || join_type == JoinType::RightAnti) - && side_type == SideType::Left) -} - -const fn is_semi(join_type: JoinTypePrimitive) -> bool { - join_type == JoinType::LeftSemi || join_type == JoinType::RightSemi -} - -const fn is_anti(join_type: JoinTypePrimitive) -> bool { - join_type == JoinType::LeftAnti || join_type == JoinType::RightAnti -} - -const fn is_left_semi_or_anti(join_type: JoinTypePrimitive) -> bool { - join_type == JoinType::LeftSemi || join_type == JoinType::LeftAnti -} - -const fn is_right_semi_or_anti(join_type: JoinTypePrimitive) -> bool { - join_type == JoinType::RightSemi || join_type == JoinType::RightAnti -} - -const fn need_left_degree(join_type: JoinTypePrimitive) -> bool { - join_type == FullOuter - || join_type == LeftOuter - || join_type == LeftAnti - || join_type == LeftSemi -} - -const fn need_right_degree(join_type: JoinTypePrimitive) -> bool { - join_type == FullOuter - || join_type == RightOuter - || join_type == RightAnti - || join_type == RightSemi -} - fn is_subset(vec1: Vec, vec2: Vec) -> bool { HashSet::::from_iter(vec1).is_subset(&vec2.into_iter().collect()) } @@ -229,9 +141,9 @@ pub struct HashJoinExecutor, + input_l: Option, /// Right input executor - input_r: Option, + input_r: Option, /// The data types of the formed new columns actual_output_data_types: Vec, /// The parameters of the left join executor @@ -277,26 +189,10 @@ impl std::fmt::Debug } } -impl Executor for HashJoinExecutor { +impl Execute for HashJoinExecutor { fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } -} - -struct HashJoinChunkBuilder { - stream_chunk_builder: JoinStreamChunkBuilder, } struct EqJoinArgs<'a, K: HashKey, S: StateStore> { @@ -312,128 +208,13 @@ struct EqJoinArgs<'a, K: HashKey, S: StateStore> { cnt_rows_received: &'a mut u32, } -impl HashJoinChunkBuilder { - fn with_match_on_insert( - &mut self, - row: &RowRef<'_>, - matched_row: &JoinRow, - ) -> Option { - // Left/Right Anti sides - if is_anti(T) { - if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { - self.stream_chunk_builder - .append_row_matched(Op::Delete, &matched_row.row) - } else { - None - } - // Left/Right Semi sides - } else if is_semi(T) { - if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { - self.stream_chunk_builder - .append_row_matched(Op::Insert, &matched_row.row) - } else { - None - } - // Outer sides - } else if matched_row.is_zero_degree() && outer_side_null(T, SIDE) { - // if the matched_row does not have any current matches - // `StreamChunkBuilder` guarantees that `UpdateDelete` will never - // issue an output chunk. - if self - .stream_chunk_builder - .append_row_matched(Op::UpdateDelete, &matched_row.row) - .is_some() - { - unreachable!("`Op::UpdateDelete` should not yield chunk"); - } - self.stream_chunk_builder - .append_row(Op::UpdateInsert, row, &matched_row.row) - // Inner sides - } else { - self.stream_chunk_builder - .append_row(Op::Insert, row, &matched_row.row) - } - } - - fn with_match_on_delete( - &mut self, - row: &RowRef<'_>, - matched_row: &JoinRow, - ) -> Option { - // Left/Right Anti sides - if is_anti(T) { - if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { - self.stream_chunk_builder - .append_row_matched(Op::Insert, &matched_row.row) - } else { - None - } - // Left/Right Semi sides - } else if is_semi(T) { - if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { - self.stream_chunk_builder - .append_row_matched(Op::Delete, &matched_row.row) - } else { - None - } - // Outer sides - } else if matched_row.is_zero_degree() && outer_side_null(T, SIDE) { - // if the matched_row does not have any current - // matches - if self - .stream_chunk_builder - .append_row(Op::UpdateDelete, row, &matched_row.row) - .is_some() - { - unreachable!("`Op::UpdateDelete` should not yield chunk"); - } - self.stream_chunk_builder - .append_row_matched(Op::UpdateInsert, &matched_row.row) - // Inner sides - } else { - // concat with the matched_row and append the new - // row - // FIXME: we always use `Op::Delete` here to avoid - // violating - // the assumption for U+ after U-. - self.stream_chunk_builder - .append_row(Op::Delete, row, &matched_row.row) - } - } - - #[inline] - fn forward_exactly_once_if_matched(&mut self, op: Op, row: RowRef<'_>) -> Option { - // if it's a semi join and the side needs to be maintained. - if is_semi(T) && forward_exactly_once(T, SIDE) { - self.stream_chunk_builder.append_row_update(op, row) - } else { - None - } - } - - #[inline] - fn forward_if_not_matched(&mut self, op: Op, row: RowRef<'_>) -> Option { - // if it's outer join or anti join and the side needs to be maintained. - if (is_anti(T) && forward_exactly_once(T, SIDE)) || is_outer_side(T, SIDE) { - self.stream_chunk_builder.append_row_update(op, row) - } else { - None - } - } - - #[inline] - fn take(&mut self) -> Option { - self.stream_chunk_builder.take() - } -} - impl HashJoinExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, info: ExecutorInfo, - input_l: BoxedExecutor, - input_r: BoxedExecutor, + input_l: Executor, + input_r: Executor, params_l: JoinParams, params_r: JoinParams, null_safe: Vec, @@ -1025,14 +806,13 @@ impl HashJoinExecutor { - stream_chunk_builder: JoinStreamChunkBuilder::new( + let mut hashjoin_chunk_builder = + JoinChunkBuilder::::new(JoinStreamChunkBuilder::new( chunk_size, actual_output_data_types.to_vec(), side_update.i2o_mapping.clone(), side_match.i2o_mapping.clone(), - ), - }; + )); let join_matched_join_keys = ctx .streaming_metrics @@ -1043,7 +823,7 @@ impl HashJoinExecutor::new( - ActorContext::create(123), + ActorContext::for_test(123), info, - Box::new(source_l), - Box::new(source_r), + source_l, + source_r, params_l, params_r, vec![null_safe], @@ -1424,7 +1206,7 @@ mod tests { Arc::new(StreamingMetrics::unused()), 1024, ); - (tx_l, tx_r, Box::new(executor).execute()) + (tx_l, tx_r, executor.boxed().execute()) } async fn create_classical_executor( @@ -1445,8 +1227,10 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (tx_l, source_l) = MockSource::channel(schema.clone(), vec![0]); - let (tx_r, source_r) = MockSource::channel(schema, vec![0]); + let (tx_l, source_l) = MockSource::channel(); + let source_l = source_l.into_executor(schema.clone(), vec![0]); + let (tx_r, source_r) = MockSource::channel(); + let source_r = source_r.into_executor(schema, vec![0]); let params_l = JoinParams::new(vec![0, 1], vec![]); let params_r = JoinParams::new(vec![0, 1], vec![]); let cond = with_condition.then(|| create_cond(None)); @@ -1495,10 +1279,10 @@ mod tests { }; let executor = HashJoinExecutor::::new( - ActorContext::create(123), + ActorContext::for_test(123), info, - Box::new(source_l), - Box::new(source_r), + source_l, + source_r, params_l, params_r, vec![false], @@ -1514,7 +1298,7 @@ mod tests { Arc::new(StreamingMetrics::unused()), 1024, ); - (tx_l, tx_r, Box::new(executor).execute()) + (tx_l, tx_r, executor.boxed().execute()) } #[tokio::test] diff --git a/src/stream/src/executor/hop_window.rs b/src/stream/src/executor/hop_window.rs index 6c6f7b99a8876..801f3daa9f53a 100644 --- a/src/stream/src/executor/hop_window.rs +++ b/src/stream/src/executor/hop_window.rs @@ -23,13 +23,12 @@ use risingwave_expr::expr::NonStrictExpression; use risingwave_expr::ExprError; use super::error::StreamExecutorError; -use super::{ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message}; +use super::{ActorContextRef, Execute, Executor, Message}; use crate::common::StreamChunkBuilder; pub struct HopWindowExecutor { _ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, pub time_col_idx: usize, pub window_slide: Interval, pub window_size: Interval, @@ -43,8 +42,7 @@ impl HopWindowExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, time_col_idx: usize, window_slide: Interval, window_size: Interval, @@ -55,7 +53,6 @@ impl HopWindowExecutor { ) -> Self { HopWindowExecutor { _ctx: ctx, - info, input, time_col_idx, window_slide, @@ -68,22 +65,10 @@ impl HopWindowExecutor { } } -impl Executor for HopWindowExecutor { +impl Execute for HopWindowExecutor { fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } impl HopWindowExecutor { @@ -253,12 +238,13 @@ mod tests { use risingwave_expr::expr::test_utils::make_hop_window_expression; use risingwave_expr::expr::NonStrictExpression; + use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Executor, ExecutorInfo, StreamChunk}; + use crate::executor::{ActorContext, Execute, StreamChunk}; const CHUNK_SIZE: usize = 256; - fn create_executor(output_indices: Vec) -> Box { + fn create_executor(output_indices: Vec) -> Box { let field1 = Field::unnamed(DataType::Int64); let field2 = Field::unnamed(DataType::Int64); let field3 = Field::with_name(DataType::Timestamp, "created_at"); @@ -278,7 +264,7 @@ mod tests { .replace('^', "2022-02-02T"), ); let input = - MockSource::with_chunks(schema.clone(), pk_indices.clone(), vec![chunk]).boxed(); + MockSource::with_chunks(vec![chunk]).into_executor(schema.clone(), pk_indices.clone()); let window_slide = Interval::from_minutes(15); let window_size = Interval::from_minutes(30); let window_offset = Interval::from_minutes(0); @@ -291,14 +277,8 @@ mod tests { ) .unwrap(); - super::HopWindowExecutor::new( - ActorContext::create(123), - ExecutorInfo { - // TODO: the schema is incorrect, but it seems useless here. - schema, - pk_indices, - identity: "HopWindowExecutor".to_string(), - }, + HopWindowExecutor::new( + ActorContext::for_test(123), input, 2, window_slide, diff --git a/src/stream/src/executor/integration_tests.rs b/src/stream/src/executor/integration_tests.rs index 65c347cbaf0ac..39e6ccada517c 100644 --- a/src/stream/src/executor/integration_tests.rs +++ b/src/stream/src/executor/integration_tests.rs @@ -36,8 +36,8 @@ use crate::executor::receiver::ReceiverExecutor; use crate::executor::test_utils::agg_executor::{ generate_agg_schema, new_boxed_simple_agg_executor, }; -use crate::executor::{Executor, MergeExecutor, ProjectExecutor, StatelessSimpleAggExecutor}; -use crate::task::SharedContext; +use crate::executor::{Execute, MergeExecutor, ProjectExecutor, StatelessSimpleAggExecutor}; +use crate::task::{LocalBarrierManager, SharedContext}; /// This test creates a merger-dispatcher pair, and run a sum. Each chunk /// has 0~9 elements. We first insert the 10 chunks, then delete them, @@ -48,43 +48,40 @@ async fn test_merger_sum_aggr() { time_zone: String::from("UTC"), }; - let actor_ctx = ActorContext::create(0); + let actor_ctx = ActorContext::for_test(0); // `make_actor` build an actor to do local aggregation let make_actor = |input_rx| { - let _schema = Schema { + let input_schema = Schema { fields: vec![Field::unnamed(DataType::Int64)], }; - let input = ReceiverExecutor::for_test(input_rx); + let input = Executor::new( + ExecutorInfo { + schema: input_schema, + pk_indices: PkIndices::new(), + identity: "ReceiverExecutor".to_string(), + }, + ReceiverExecutor::for_test(input_rx).boxed(), + ); let agg_calls = vec![ AggCall::from_pretty("(count:int8)"), AggCall::from_pretty("(sum:int8 $0:int8)"), ]; let schema = generate_agg_schema(&input, &agg_calls, None); // for the local aggregator, we need two states: row count and sum - let aggregator = StatelessSimpleAggExecutor::new( - actor_ctx.clone(), - ExecutorInfo { - schema, - pk_indices: vec![], - identity: format!("StatelessSimpleAggExecutor {:X}", 1), - }, - input.boxed(), - agg_calls, - ) - .unwrap(); + let aggregator = + StatelessSimpleAggExecutor::new(actor_ctx.clone(), input, schema, agg_calls).unwrap(); let (tx, rx) = channel_for_test(); let consumer = SenderConsumer { input: aggregator.boxed(), channel: Box::new(LocalOutput::new(233, tx)), }; - let context = SharedContext::for_test().into(); let actor = Actor::new( consumer, vec![], - context, StreamingMetrics::unused().into(), actor_ctx.clone(), expr_context.clone(), + LocalBarrierManager::for_test(), ); (actor, rx) }; @@ -110,13 +107,15 @@ async fn test_merger_sum_aggr() { // create a round robin dispatcher, which dispatches messages to the actors let (input, rx) = channel_for_test(); - let schema = Schema { - fields: vec![ - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - ], - }; - let receiver_op = Box::new(ReceiverExecutor::for_test(rx)); + let receiver_op = Executor::new( + ExecutorInfo { + // input schema of local simple agg + schema: Schema::new(vec![Field::unnamed(DataType::Int64)]), + pk_indices: PkIndices::new(), + identity: "ReceiverExecutor".to_string(), + }, + ReceiverExecutor::for_test(rx).boxed(), + ); let dispatcher = DispatchExecutor::new( receiver_op, vec![DispatcherImpl::RoundRobin(RoundRobinDataDispatcher::new( @@ -129,26 +128,36 @@ async fn test_merger_sum_aggr() { ctx, metrics, ); - let context = SharedContext::for_test().into(); let actor = Actor::new( dispatcher, vec![], - context, StreamingMetrics::unused().into(), actor_ctx.clone(), expr_context.clone(), + LocalBarrierManager::for_test(), ); handles.push(tokio::spawn(actor.run())); // use a merge operator to collect data from dispatchers before sending them to aggregator - let merger = MergeExecutor::for_test(outputs, schema); + let merger = Executor::new( + ExecutorInfo { + // output schema of local simple agg + schema: Schema::new(vec![ + Field::unnamed(DataType::Int64), + Field::unnamed(DataType::Int64), + ]), + pk_indices: PkIndices::new(), + identity: "MergeExecutor".to_string(), + }, + MergeExecutor::for_test(outputs).boxed(), + ); // for global aggregator, we need to sum data and sum row count let is_append_only = false; let aggregator = new_boxed_simple_agg_executor( actor_ctx.clone(), MemoryStateStore::new(), - merger.boxed(), + merger, is_append_only, vec![ AggCall::from_pretty("(sum0:int8 $0:int8)"), @@ -163,13 +172,6 @@ async fn test_merger_sum_aggr() { let projection = ProjectExecutor::new( actor_ctx.clone(), - ExecutorInfo { - schema: Schema { - fields: vec![Field::unnamed(DataType::Int64)], - }, - pk_indices: vec![], - identity: format!("ProjectExecutor {:X}", 3), - }, aggregator, vec![ // TODO: use the new streaming_if_null expression here, and add `None` tests @@ -185,14 +187,13 @@ async fn test_merger_sum_aggr() { input: projection.boxed(), data: items.clone(), }; - let context = SharedContext::for_test().into(); let actor = Actor::new( consumer, vec![], - context, StreamingMetrics::unused().into(), actor_ctx.clone(), expr_context.clone(), + LocalBarrierManager::for_test(), ); handles.push(tokio::spawn(actor.run())); @@ -236,7 +237,7 @@ async fn test_merger_sum_aggr() { } struct MockConsumer { - input: BoxedExecutor, + input: Box, data: Arc>>, } @@ -263,7 +264,7 @@ impl StreamConsumer for MockConsumer { /// `SenderConsumer` consumes data from input executor and send it into a channel. pub struct SenderConsumer { - input: BoxedExecutor, + input: Box, channel: BoxedOutput, } diff --git a/src/stream/src/executor/join/builder.rs b/src/stream/src/executor/join/builder.rs new file mode 100644 index 0000000000000..ca34ba9cad9e8 --- /dev/null +++ b/src/stream/src/executor/join/builder.rs @@ -0,0 +1,276 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder; +use risingwave_common::array::{Op, RowRef, StreamChunk}; +use risingwave_common::row::{OwnedRow, Row}; +use risingwave_common::types::{DataType, DatumRef}; + +use self::row::JoinRow; +// Re-export `StreamChunkBuilder`. +use super::*; +use super::{JoinTypePrimitive, SideTypePrimitive}; + +type IndexMappings = Vec<(usize, usize)>; + +/// Build stream chunks with fixed chunk size from joined two sides of rows. +pub struct JoinStreamChunkBuilder { + builder: StreamChunkBuilder, + + /// The column index mapping from update side to output. + update_to_output: IndexMappings, + + /// The column index mapping from matched side to output. + matched_to_output: IndexMappings, +} + +impl JoinStreamChunkBuilder { + pub fn new( + chunk_size: usize, + data_types: Vec, + update_to_output: IndexMappings, + matched_to_output: IndexMappings, + ) -> Self { + Self { + builder: StreamChunkBuilder::new(chunk_size, data_types), + update_to_output, + matched_to_output, + } + } + + /// Get the mappings from left/right input indices to the output indices. The mappings can be + /// used to create [`JoinStreamChunkBuilder`] later. + /// + /// Please note the semantics of `update` and `matched` when creating the builder: either left + /// or right side can be `update` side or `matched` side, the key is to call the corresponding + /// append method once you passed `left_to_output`/`right_to_output` to + /// `update_to_output`/`matched_to_output`. + pub fn get_i2o_mapping( + output_indices: &[usize], + left_len: usize, + right_len: usize, + ) -> (IndexMappings, IndexMappings) { + let mut left_to_output = vec![]; + let mut right_to_output = vec![]; + + for (output_idx, &idx) in output_indices.iter().enumerate() { + if idx < left_len { + left_to_output.push((idx, output_idx)) + } else if idx >= left_len && idx < left_len + right_len { + right_to_output.push((idx - left_len, output_idx)); + } else { + unreachable!("output_indices out of bound") + } + } + (left_to_output, right_to_output) + } + + /// Append a row with coming update value and matched value. + /// + /// A [`StreamChunk`] will be returned when `size == capacity`. + #[must_use] + pub fn append_row( + &mut self, + op: Op, + row_update: impl Row, + row_matched: impl Row, + ) -> Option { + self.builder.append_iter( + op, + self.update_to_output + .iter() + .map(|&(update_idx, output_idx)| (output_idx, row_update.datum_at(update_idx))) + .chain( + self.matched_to_output + .iter() + .map(|&(matched_idx, output_idx)| { + (output_idx, row_matched.datum_at(matched_idx)) + }), + ), + ) + } + + /// Append a row with coming update value and fill the other side with null. + /// + /// A [`StreamChunk`] will be returned when `size == capacity`. + #[must_use] + pub fn append_row_update(&mut self, op: Op, row_update: impl Row) -> Option { + self.builder.append_iter( + op, + self.update_to_output + .iter() + .map(|&(update_idx, output_idx)| (output_idx, row_update.datum_at(update_idx))) + .chain( + self.matched_to_output + .iter() + .map(|&(_, output_idx)| (output_idx, DatumRef::None)), + ), + ) + } + + /// Append a row with matched value and fill the coming side with null. + /// + /// A [`StreamChunk`] will be returned when `size == capacity`. + #[must_use] + pub fn append_row_matched(&mut self, op: Op, row_matched: impl Row) -> Option { + self.builder.append_iter( + op, + self.update_to_output + .iter() + .map(|&(_, output_idx)| (output_idx, DatumRef::None)) + .chain( + self.matched_to_output + .iter() + .map(|&(matched_idx, output_idx)| { + (output_idx, row_matched.datum_at(matched_idx)) + }), + ), + ) + } + + /// Take out the remaining rows as a chunk. Return `None` if the builder is empty. + #[must_use] + pub fn take(&mut self) -> Option { + self.builder.take() + } +} + +pub struct JoinChunkBuilder { + stream_chunk_builder: JoinStreamChunkBuilder, +} + +impl JoinChunkBuilder { + pub fn new(stream_chunk_builder: JoinStreamChunkBuilder) -> Self { + Self { + stream_chunk_builder, + } + } + + pub fn with_match_on_insert( + &mut self, + row: &RowRef<'_>, + matched_row: &JoinRow, + ) -> Option { + // Left/Right Anti sides + if is_anti(T) { + if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { + self.stream_chunk_builder + .append_row_matched(Op::Delete, &matched_row.row) + } else { + None + } + // Left/Right Semi sides + } else if is_semi(T) { + if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { + self.stream_chunk_builder + .append_row_matched(Op::Insert, &matched_row.row) + } else { + None + } + // Outer sides + } else if matched_row.is_zero_degree() && outer_side_null(T, SIDE) { + // if the matched_row does not have any current matches + // `StreamChunkBuilder` guarantees that `UpdateDelete` will never + // issue an output chunk. + if self + .stream_chunk_builder + .append_row_matched(Op::UpdateDelete, &matched_row.row) + .is_some() + { + unreachable!("`Op::UpdateDelete` should not yield chunk"); + } + self.stream_chunk_builder + .append_row(Op::UpdateInsert, row, &matched_row.row) + // Inner sides + } else { + self.stream_chunk_builder + .append_row(Op::Insert, row, &matched_row.row) + } + } + + pub fn with_match_on_delete( + &mut self, + row: &RowRef<'_>, + matched_row: &JoinRow, + ) -> Option { + // Left/Right Anti sides + if is_anti(T) { + if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { + self.stream_chunk_builder + .append_row_matched(Op::Insert, &matched_row.row) + } else { + None + } + // Left/Right Semi sides + } else if is_semi(T) { + if matched_row.is_zero_degree() && only_forward_matched_side(T, SIDE) { + self.stream_chunk_builder + .append_row_matched(Op::Delete, &matched_row.row) + } else { + None + } + // Outer sides + } else if matched_row.is_zero_degree() && outer_side_null(T, SIDE) { + // if the matched_row does not have any current + // matches + if self + .stream_chunk_builder + .append_row(Op::UpdateDelete, row, &matched_row.row) + .is_some() + { + unreachable!("`Op::UpdateDelete` should not yield chunk"); + } + self.stream_chunk_builder + .append_row_matched(Op::UpdateInsert, &matched_row.row) + // Inner sides + } else { + // concat with the matched_row and append the new + // row + // FIXME: we always use `Op::Delete` here to avoid + // violating + // the assumption for U+ after U-. + self.stream_chunk_builder + .append_row(Op::Delete, row, &matched_row.row) + } + } + + #[inline] + pub fn forward_exactly_once_if_matched( + &mut self, + op: Op, + row: RowRef<'_>, + ) -> Option { + // if it's a semi join and the side needs to be maintained. + if is_semi(T) && forward_exactly_once(T, SIDE) { + self.stream_chunk_builder.append_row_update(op, row) + } else { + None + } + } + + #[inline] + pub fn forward_if_not_matched(&mut self, op: Op, row: RowRef<'_>) -> Option { + // if it's outer join or anti join and the side needs to be maintained. + if (is_anti(T) && forward_exactly_once(T, SIDE)) || is_outer_side(T, SIDE) { + self.stream_chunk_builder.append_row_update(op, row) + } else { + None + } + } + + #[inline] + pub fn take(&mut self) -> Option { + self.stream_chunk_builder.take() + } +} diff --git a/src/stream/src/executor/join/hash_join.rs b/src/stream/src/executor/join/hash_join.rs new file mode 100644 index 0000000000000..123bd6e42e45e --- /dev/null +++ b/src/stream/src/executor/join/hash_join.rs @@ -0,0 +1,783 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::alloc::Global; +use std::ops::{Bound, Deref, DerefMut}; +use std::sync::Arc; + +use anyhow::Context; +use futures::future::{join, try_join}; +use futures::StreamExt; +use futures_async_stream::for_await; +use local_stats_alloc::{SharedStatsAlloc, StatsAlloc}; +use risingwave_common::buffer::Bitmap; +use risingwave_common::estimate_size::EstimateSize; +use risingwave_common::hash::{HashKey, PrecomputedBuildHasher}; +use risingwave_common::metrics::LabelGuardedIntCounter; +use risingwave_common::row::{OwnedRow, Row, RowExt}; +use risingwave_common::types::{DataType, ScalarImpl}; +use risingwave_common::util::epoch::EpochPair; +use risingwave_common::util::row_serde::OrderedRowSerde; +use risingwave_common::util::sort_util::OrderType; +use risingwave_storage::store::PrefetchOptions; +use risingwave_storage::StateStore; + +use super::row::{DegreeType, EncodedJoinRow}; +use crate::cache::{new_with_hasher_in, ManagedLruCache}; +use crate::common::metrics::MetricsInfo; +use crate::common::table::state_table::StateTable; +use crate::executor::error::StreamExecutorResult; +use crate::executor::join::row::JoinRow; +use crate::executor::monitor::StreamingMetrics; +use crate::task::{ActorId, AtomicU64Ref, FragmentId}; + +/// Memcomparable encoding. +type PkType = Vec; + +pub type StateValueType = EncodedJoinRow; +pub type HashValueType = Box; + +impl EstimateSize for HashValueType { + fn estimated_heap_size(&self) -> usize { + self.as_ref().estimated_heap_size() + } +} + +/// The wrapper for [`JoinEntryState`] which should be `Some` most of the time in the hash table. +/// +/// When the executor is operating on the specific entry of the map, it can hold the ownership of +/// the entry by taking the value out of the `Option`, instead of holding a mutable reference to the +/// map, which can make the compiler happy. +struct HashValueWrapper(Option); + +impl EstimateSize for HashValueWrapper { + fn estimated_heap_size(&self) -> usize { + self.0.estimated_heap_size() + } +} + +impl HashValueWrapper { + const MESSAGE: &'static str = "the state should always be `Some`"; + + /// Take the value out of the wrapper. Panic if the value is `None`. + pub fn take(&mut self) -> HashValueType { + self.0.take().expect(Self::MESSAGE) + } +} + +impl Deref for HashValueWrapper { + type Target = HashValueType; + + fn deref(&self) -> &Self::Target { + self.0.as_ref().expect(Self::MESSAGE) + } +} + +impl DerefMut for HashValueWrapper { + fn deref_mut(&mut self) -> &mut Self::Target { + self.0.as_mut().expect(Self::MESSAGE) + } +} + +type JoinHashMapInner = + ManagedLruCache>; + +pub struct JoinHashMapMetrics { + /// Basic information + /// How many times have we hit the cache of join executor + lookup_miss_count: usize, + total_lookup_count: usize, + /// How many times have we miss the cache when insert row + insert_cache_miss_count: usize, + + // Metrics + join_lookup_total_count_metric: LabelGuardedIntCounter<5>, + join_lookup_miss_count_metric: LabelGuardedIntCounter<5>, + join_insert_cache_miss_count_metrics: LabelGuardedIntCounter<5>, +} + +impl JoinHashMapMetrics { + pub fn new( + metrics: &StreamingMetrics, + actor_id: ActorId, + fragment_id: FragmentId, + side: &'static str, + join_table_id: u32, + degree_table_id: u32, + ) -> Self { + let actor_id = actor_id.to_string(); + let fragment_id = fragment_id.to_string(); + let join_table_id = join_table_id.to_string(); + let degree_table_id = degree_table_id.to_string(); + let join_lookup_total_count_metric = + metrics.join_lookup_total_count.with_guarded_label_values(&[ + (side), + &join_table_id, + °ree_table_id, + &actor_id, + &fragment_id, + ]); + let join_lookup_miss_count_metric = + metrics.join_lookup_miss_count.with_guarded_label_values(&[ + (side), + &join_table_id, + °ree_table_id, + &actor_id, + &fragment_id, + ]); + let join_insert_cache_miss_count_metrics = metrics + .join_insert_cache_miss_count + .with_guarded_label_values(&[ + (side), + &join_table_id, + °ree_table_id, + &actor_id, + &fragment_id, + ]); + + Self { + lookup_miss_count: 0, + total_lookup_count: 0, + insert_cache_miss_count: 0, + join_lookup_total_count_metric, + join_lookup_miss_count_metric, + join_insert_cache_miss_count_metrics, + } + } + + pub fn flush(&mut self) { + self.join_lookup_total_count_metric + .inc_by(self.total_lookup_count as u64); + self.join_lookup_miss_count_metric + .inc_by(self.lookup_miss_count as u64); + self.join_insert_cache_miss_count_metrics + .inc_by(self.insert_cache_miss_count as u64); + self.total_lookup_count = 0; + self.lookup_miss_count = 0; + self.insert_cache_miss_count = 0; + } +} + +pub struct JoinHashMap { + /// Store the join states. + inner: JoinHashMapInner, + /// Data types of the join key columns + join_key_data_types: Vec, + /// Null safe bitmap for each join pair + null_matched: K::Bitmap, + /// The memcomparable serializer of primary key. + pk_serializer: OrderedRowSerde, + /// State table. Contains the data from upstream. + state: TableInner, + /// Degree table. + /// + /// The degree is generated from the hash join executor. + /// Each row in `state` has a corresponding degree in `degree state`. + /// A degree value `d` in for a row means the row has `d` matched row in the other join side. + /// + /// It will only be used when needed in a side. + /// + /// - Full Outer: both side + /// - Left Outer/Semi/Anti: left side + /// - Right Outer/Semi/Anti: right side + /// - Inner: None. + degree_state: TableInner, + /// If degree table is need + need_degree_table: bool, + /// Pk is part of the join key. + pk_contained_in_jk: bool, + /// Metrics of the hash map + metrics: JoinHashMapMetrics, +} + +struct TableInner { + /// Indices of the (cache) pk in a state row + pk_indices: Vec, + /// Indices of the join key in a state row + join_key_indices: Vec, + // This should be identical to the pk in state table. + order_key_indices: Vec, + // This should be identical to the data types in table schema. + #[expect(dead_code)] + all_data_types: Vec, + pub(crate) table: StateTable, +} + +impl TableInner { + fn error_context(&self, row: &impl Row) -> String { + let pk = row.project(&self.pk_indices); + let jk = row.project(&self.join_key_indices); + format!( + "join key: {}, pk: {}, row: {}, state_table_id: {}", + jk.display(), + pk.display(), + row.display(), + self.table.table_id() + ) + } +} + +impl JoinHashMap { + /// Create a [`JoinHashMap`] with the given LRU capacity. + #[allow(clippy::too_many_arguments)] + pub fn new( + watermark_epoch: AtomicU64Ref, + join_key_data_types: Vec, + state_join_key_indices: Vec, + state_all_data_types: Vec, + state_table: StateTable, + state_pk_indices: Vec, + degree_join_key_indices: Vec, + degree_all_data_types: Vec, + degree_table: StateTable, + degree_pk_indices: Vec, + null_matched: K::Bitmap, + need_degree_table: bool, + pk_contained_in_jk: bool, + metrics: Arc, + actor_id: ActorId, + fragment_id: FragmentId, + side: &'static str, + ) -> Self { + let alloc = StatsAlloc::new(Global).shared(); + // TODO: unify pk encoding with state table. + let pk_data_types = state_pk_indices + .iter() + .map(|i| state_all_data_types[*i].clone()) + .collect(); + let pk_serializer = OrderedRowSerde::new( + pk_data_types, + vec![OrderType::ascending(); state_pk_indices.len()], + ); + + let join_table_id = state_table.table_id(); + let degree_table_id = degree_table.table_id(); + let state = TableInner { + pk_indices: state_pk_indices, + join_key_indices: state_join_key_indices, + order_key_indices: state_table.pk_indices().to_vec(), + all_data_types: state_all_data_types, + table: state_table, + }; + + let degree_state = TableInner { + pk_indices: degree_pk_indices, + join_key_indices: degree_join_key_indices, + order_key_indices: degree_table.pk_indices().to_vec(), + all_data_types: degree_all_data_types, + table: degree_table, + }; + + let metrics_info = MetricsInfo::new( + metrics.clone(), + join_table_id, + actor_id, + &format!("hash join {}", side), + ); + + let cache = + new_with_hasher_in(watermark_epoch, metrics_info, PrecomputedBuildHasher, alloc); + + Self { + inner: cache, + join_key_data_types, + null_matched, + pk_serializer, + state, + degree_state, + need_degree_table, + pk_contained_in_jk, + metrics: JoinHashMapMetrics::new( + &metrics, + actor_id, + fragment_id, + side, + join_table_id, + degree_table_id, + ), + } + } + + pub fn init(&mut self, epoch: EpochPair) { + self.update_epoch(epoch.curr); + self.state.table.init_epoch(epoch); + self.degree_state.table.init_epoch(epoch); + } + + pub fn update_epoch(&mut self, epoch: u64) { + // Update the current epoch in `ManagedLruCache` + self.inner.update_epoch(epoch) + } + + /// Update the vnode bitmap and manipulate the cache if necessary. + pub fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc) -> bool { + let (_previous_vnode_bitmap, cache_may_stale) = + self.state.table.update_vnode_bitmap(vnode_bitmap.clone()); + let _ = self.degree_state.table.update_vnode_bitmap(vnode_bitmap); + + if cache_may_stale { + self.inner.clear(); + } + + cache_may_stale + } + + pub fn update_watermark(&mut self, watermark: ScalarImpl) { + // TODO: remove data in cache. + self.state.table.update_watermark(watermark.clone(), false); + self.degree_state.table.update_watermark(watermark, false); + } + + /// Take the state for the given `key` out of the hash table and return it. One **MUST** call + /// `update_state` after some operations to put the state back. + /// + /// If the state does not exist in the cache, fetch the remote storage and return. If it still + /// does not exist in the remote storage, a [`JoinEntryState`] with empty cache will be + /// returned. + /// + /// Note: This will NOT remove anything from remote storage. + pub async fn take_state<'a>(&mut self, key: &K) -> StreamExecutorResult { + self.metrics.total_lookup_count += 1; + let state = if self.inner.contains(key) { + // Do not update the LRU statistics here with `peek_mut` since we will put the state + // back. + let mut state = self.inner.peek_mut(key).unwrap(); + state.take() + } else { + self.metrics.lookup_miss_count += 1; + self.fetch_cached_state(key).await?.into() + }; + Ok(state) + } + + /// Fetch cache from the state store. Should only be called if the key does not exist in memory. + /// Will return a empty `JoinEntryState` even when state does not exist in remote. + async fn fetch_cached_state(&self, key: &K) -> StreamExecutorResult { + let key = key.deserialize(&self.join_key_data_types)?; + + let mut entry_state = JoinEntryState::default(); + + if self.need_degree_table { + let sub_range: &(Bound, Bound) = + &(Bound::Unbounded, Bound::Unbounded); + let table_iter_fut = + self.state + .table + .iter_with_prefix(&key, sub_range, PrefetchOptions::default()); + let degree_table_iter_fut = self.degree_state.table.iter_with_prefix( + &key, + sub_range, + PrefetchOptions::default(), + ); + + let (table_iter, degree_table_iter) = + try_join(table_iter_fut, degree_table_iter_fut).await?; + + let mut pinned_table_iter = std::pin::pin!(table_iter); + let mut pinned_degree_table_iter = std::pin::pin!(degree_table_iter); + loop { + // Iterate on both iterators and ensure they have same size. Basically `zip_eq()`. + let (row, degree) = + join(pinned_table_iter.next(), pinned_degree_table_iter.next()).await; + let (row, degree) = match (row, degree) { + (None, None) => break, + (None, Some(_)) | (Some(_), None) => { + panic!("mismatched row and degree table of join key: {:?}", &key) + } + (Some(r), Some(d)) => (r, d), + }; + + let row = row?; + let degree_row = degree?; + let pk1 = row.key(); + let pk2 = degree_row.key(); + debug_assert_eq!( + pk1, pk2, + "mismatched pk in degree table: pk1: {pk1:?}, pk2: {pk2:?}", + ); + let pk = row + .as_ref() + .project(&self.state.pk_indices) + .memcmp_serialize(&self.pk_serializer); + let degree_i64 = degree_row + .datum_at(degree_row.len() - 1) + .expect("degree should not be NULL"); + entry_state + .insert( + pk, + JoinRow::new(row.row(), degree_i64.into_int64() as u64).encode(), + ) + .with_context(|| self.state.error_context(row.row()))?; + } + } else { + let sub_range: &(Bound, Bound) = + &(Bound::Unbounded, Bound::Unbounded); + let table_iter = self + .state + .table + .iter_with_prefix(&key, sub_range, PrefetchOptions::default()) + .await?; + + #[for_await] + for entry in table_iter { + let row = entry?; + let pk = row + .as_ref() + .project(&self.state.pk_indices) + .memcmp_serialize(&self.pk_serializer); + entry_state + .insert(pk, JoinRow::new(row.row(), 0).encode()) + .with_context(|| self.state.error_context(row.row()))?; + } + }; + + Ok(entry_state) + } + + pub async fn flush(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { + self.metrics.flush(); + self.state.table.commit(epoch).await?; + self.degree_state.table.commit(epoch).await?; + Ok(()) + } + + pub async fn try_flush(&mut self) -> StreamExecutorResult<()> { + self.state.table.try_flush().await?; + self.degree_state.table.try_flush().await?; + Ok(()) + } + + /// Insert a join row + #[allow(clippy::unused_async)] + pub async fn insert(&mut self, key: &K, value: JoinRow) -> StreamExecutorResult<()> { + let pk = (&value.row) + .project(&self.state.pk_indices) + .memcmp_serialize(&self.pk_serializer); + + // TODO(yuhao): avoid this `contains`. + // https://github.com/risingwavelabs/risingwave/issues/9233 + if self.inner.contains(key) { + // Update cache + let mut entry = self.inner.get_mut(key).unwrap(); + entry + .insert(pk, value.encode()) + .with_context(|| self.state.error_context(&value.row))?; + } else if self.pk_contained_in_jk { + // Refill cache when the join key exist in neither cache or storage. + self.metrics.insert_cache_miss_count += 1; + let mut state = JoinEntryState::default(); + state + .insert(pk, value.encode()) + .with_context(|| self.state.error_context(&value.row))?; + self.update_state(key, state.into()); + } + + // Update the flush buffer. + let (row, degree) = value.to_table_rows(&self.state.order_key_indices); + self.state.table.insert(row); + self.degree_state.table.insert(degree); + Ok(()) + } + + /// Insert a row. + /// Used when the side does not need to update degree. + #[allow(clippy::unused_async)] + pub async fn insert_row(&mut self, key: &K, value: impl Row) -> StreamExecutorResult<()> { + let join_row = JoinRow::new(&value, 0); + let pk = (&value) + .project(&self.state.pk_indices) + .memcmp_serialize(&self.pk_serializer); + + // TODO(yuhao): avoid this `contains`. + // https://github.com/risingwavelabs/risingwave/issues/9233 + if self.inner.contains(key) { + // Update cache + let mut entry = self.inner.get_mut(key).unwrap(); + entry + .insert(pk, join_row.encode()) + .with_context(|| self.state.error_context(&value))?; + } else if self.pk_contained_in_jk { + // Refill cache when the join key exist in neither cache or storage. + self.metrics.insert_cache_miss_count += 1; + let mut state = JoinEntryState::default(); + state + .insert(pk, join_row.encode()) + .with_context(|| self.state.error_context(&value))?; + self.update_state(key, state.into()); + } + + // Update the flush buffer. + self.state.table.insert(value); + Ok(()) + } + + /// Delete a join row + pub fn delete(&mut self, key: &K, value: JoinRow) -> StreamExecutorResult<()> { + if let Some(mut entry) = self.inner.get_mut(key) { + let pk = (&value.row) + .project(&self.state.pk_indices) + .memcmp_serialize(&self.pk_serializer); + entry + .remove(pk) + .with_context(|| self.state.error_context(&value.row))?; + } + + // If no cache maintained, only update the state table. + let (row, degree) = value.to_table_rows(&self.state.order_key_indices); + self.state.table.delete(row); + self.degree_state.table.delete(degree); + Ok(()) + } + + /// Delete a row + /// Used when the side does not need to update degree. + pub fn delete_row(&mut self, key: &K, value: impl Row) -> StreamExecutorResult<()> { + if let Some(mut entry) = self.inner.get_mut(key) { + let pk = (&value) + .project(&self.state.pk_indices) + .memcmp_serialize(&self.pk_serializer); + entry + .remove(pk) + .with_context(|| self.state.error_context(&value))?; + } + + // If no cache maintained, only update the state table. + self.state.table.delete(value); + Ok(()) + } + + /// Update a [`JoinEntryState`] into the hash table. + pub fn update_state(&mut self, key: &K, state: HashValueType) { + self.inner.put(key.clone(), HashValueWrapper(Some(state))); + } + + /// Manipulate the degree of the given [`JoinRow`] and [`EncodedJoinRow`] with `action`, both in + /// memory and in the degree table. + fn manipulate_degree( + &mut self, + join_row_ref: &mut StateValueType, + join_row: &mut JoinRow, + action: impl Fn(&mut DegreeType), + ) { + // TODO: no need to `into_owned_row` here due to partial borrow. + let old_degree = join_row + .to_table_rows(&self.state.order_key_indices) + .1 + .into_owned_row(); + + action(&mut join_row_ref.degree); + action(&mut join_row.degree); + + let new_degree = join_row.to_table_rows(&self.state.order_key_indices).1; + + self.degree_state.table.update(old_degree, new_degree); + } + + /// Increment the degree of the given [`JoinRow`] and [`EncodedJoinRow`] with `action`, both in + /// memory and in the degree table. + pub fn inc_degree( + &mut self, + join_row_ref: &mut StateValueType, + join_row: &mut JoinRow, + ) { + self.manipulate_degree(join_row_ref, join_row, |d| *d += 1) + } + + /// Decrement the degree of the given [`JoinRow`] and [`EncodedJoinRow`] with `action`, both in + /// memory and in the degree table. + pub fn dec_degree( + &mut self, + join_row_ref: &mut StateValueType, + join_row: &mut JoinRow, + ) { + self.manipulate_degree(join_row_ref, join_row, |d| { + *d = d + .checked_sub(1) + .expect("Tried to decrement zero join row degree") + }) + } + + /// Evict the cache. + pub fn evict(&mut self) { + self.inner.evict(); + } + + /// Cached entry count for this hash table. + pub fn entry_count(&self) -> usize { + self.inner.len() + } + + pub fn null_matched(&self) -> &K::Bitmap { + &self.null_matched + } + + pub fn table_id(&self) -> u32 { + self.state.table.table_id() + } +} + +use risingwave_common::estimate_size::KvSize; +use thiserror::Error; + +use super::*; + +/// We manages a `HashMap` in memory for all entries belonging to a join key. +/// When evicted, `cached` does not hold any entries. +/// +/// If a `JoinEntryState` exists for a join key, the all records under this +/// join key will be presented in the cache. +#[derive(Default)] +pub struct JoinEntryState { + /// The full copy of the state. + cached: join_row_set::JoinRowSet, + kv_heap_size: KvSize, +} + +impl EstimateSize for JoinEntryState { + fn estimated_heap_size(&self) -> usize { + // TODO: Add btreemap internal size. + // https://github.com/risingwavelabs/risingwave/issues/9713 + self.kv_heap_size.size() + } +} + +#[derive(Error, Debug)] +pub enum JoinEntryError { + #[error("double inserting a join state entry")] + OccupiedError, + #[error("removing a join state entry but it is not in the cache")] + RemoveError, +} + +impl JoinEntryState { + /// Insert into the cache. + pub fn insert( + &mut self, + key: PkType, + value: StateValueType, + ) -> Result<&mut StateValueType, JoinEntryError> { + self.kv_heap_size.add(&key, &value); + self.cached + .try_insert(key, value) + .map_err(|_| JoinEntryError::OccupiedError) + } + + /// Delete from the cache. + pub fn remove(&mut self, pk: PkType) -> Result<(), JoinEntryError> { + if let Some(value) = self.cached.remove(&pk) { + self.kv_heap_size.sub(&pk, &value); + Ok(()) + } else { + Err(JoinEntryError::RemoveError) + } + } + + /// Note: the first item in the tuple is the mutable reference to the value in this entry, while + /// the second item is the decoded value. To mutate the degree, one **must not** forget to apply + /// the changes to the first item. + /// + /// WARNING: Should not change the heap size of `StateValueType` with the mutable reference. + pub fn values_mut<'a>( + &'a mut self, + data_types: &'a [DataType], + ) -> impl Iterator< + Item = ( + &'a mut StateValueType, + StreamExecutorResult>, + ), + > + 'a { + self.cached.values_mut().map(|encoded| { + let decoded = encoded.decode(data_types); + (encoded, decoded) + }) + } + + pub fn len(&self) -> usize { + self.cached.len() + } +} + +#[cfg(test)] +mod tests { + use itertools::Itertools; + use risingwave_common::array::*; + use risingwave_common::types::{DataType, ScalarImpl}; + use risingwave_common::util::iter_util::ZipEqDebug; + + use super::*; + + fn insert_chunk( + managed_state: &mut JoinEntryState, + pk_indices: &[usize], + data_chunk: &DataChunk, + ) { + for row_ref in data_chunk.rows() { + let row: OwnedRow = row_ref.into_owned_row(); + let value_indices = (0..row.len() - 1).collect_vec(); + let pk = pk_indices.iter().map(|idx| row[*idx].clone()).collect_vec(); + // Pk is only a `i64` here, so encoding method does not matter. + let pk = OwnedRow::new(pk).project(&value_indices).value_serialize(); + let join_row = JoinRow { row, degree: 0 }; + managed_state.insert(pk, join_row.encode()).unwrap(); + } + } + + fn check( + managed_state: &mut JoinEntryState, + col_types: &[DataType], + col1: &[i64], + col2: &[i64], + ) { + for ((_, matched_row), (d1, d2)) in managed_state + .values_mut(col_types) + .zip_eq_debug(col1.iter().zip_eq_debug(col2.iter())) + { + let matched_row = matched_row.unwrap(); + assert_eq!(matched_row.row[0], Some(ScalarImpl::Int64(*d1))); + assert_eq!(matched_row.row[1], Some(ScalarImpl::Int64(*d2))); + assert_eq!(matched_row.degree, 0); + } + } + + #[tokio::test] + async fn test_managed_all_or_none_state() { + let mut managed_state = JoinEntryState::default(); + let col_types = vec![DataType::Int64, DataType::Int64]; + let pk_indices = [0]; + + let col1 = [3, 2, 1]; + let col2 = [4, 5, 6]; + let data_chunk1 = DataChunk::from_pretty( + "I I + 3 4 + 2 5 + 1 6", + ); + + // `Vec` in state + insert_chunk(&mut managed_state, &pk_indices, &data_chunk1); + check(&mut managed_state, &col_types, &col1, &col2); + + // `BtreeMap` in state + let col1 = [1, 2, 3, 4, 5]; + let col2 = [6, 5, 4, 9, 8]; + let data_chunk2 = DataChunk::from_pretty( + "I I + 5 8 + 4 9", + ); + insert_chunk(&mut managed_state, &pk_indices, &data_chunk2); + check(&mut managed_state, &col_types, &col1, &col2); + } +} diff --git a/src/stream/src/executor/join/join_row_set.rs b/src/stream/src/executor/join/join_row_set.rs new file mode 100644 index 0000000000000..de6f5ce2f0279 --- /dev/null +++ b/src/stream/src/executor/join/join_row_set.rs @@ -0,0 +1,120 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::btree_map::OccupiedError as BTreeMapOccupiedError; +use std::collections::BTreeMap; +use std::fmt::Debug; +use std::mem; + +use auto_enums::auto_enum; +use enum_as_inner::EnumAsInner; + +const MAX_VEC_SIZE: usize = 4; + +#[derive(Debug, EnumAsInner)] +pub enum JoinRowSet { + BTree(BTreeMap), + Vec(Vec<(K, V)>), +} + +impl Default for JoinRowSet { + fn default() -> Self { + Self::Vec(Vec::new()) + } +} + +#[derive(Debug)] +#[allow(dead_code)] +pub struct VecOccupiedError<'a, K, V> { + key: &'a K, + old_value: &'a V, + new_value: V, +} + +#[derive(Debug)] +pub enum JoinRowSetOccupiedError<'a, K: Ord, V> { + BTree(BTreeMapOccupiedError<'a, K, V>), + Vec(VecOccupiedError<'a, K, V>), +} + +impl JoinRowSet { + pub fn try_insert( + &mut self, + key: K, + value: V, + ) -> Result<&'_ mut V, JoinRowSetOccupiedError<'_, K, V>> { + if let Self::Vec(inner) = self + && inner.len() >= MAX_VEC_SIZE + { + let btree = BTreeMap::from_iter(inner.drain(..)); + mem::swap(self, &mut Self::BTree(btree)); + } + + match self { + Self::BTree(inner) => inner + .try_insert(key, value) + .map_err(JoinRowSetOccupiedError::BTree), + Self::Vec(inner) => { + if let Some(pos) = inner.iter().position(|elem| elem.0 == key) { + Err(JoinRowSetOccupiedError::Vec(VecOccupiedError { + key: &inner[pos].0, + old_value: &inner[pos].1, + new_value: value, + })) + } else { + if inner.capacity() == 0 { + // `Vec` will give capacity 4 when `1 < mem::size_of:: <= 1024` + // We only give one for memory optimization + inner.reserve_exact(1); + } + inner.push((key, value)); + Ok(&mut inner.last_mut().unwrap().1) + } + } + } + } + + pub fn remove(&mut self, key: &K) -> Option { + let ret = match self { + Self::BTree(inner) => inner.remove(key), + Self::Vec(inner) => inner + .iter() + .position(|elem| &elem.0 == key) + .map(|pos| inner.swap_remove(pos).1), + }; + if let Self::BTree(inner) = self + && inner.len() <= MAX_VEC_SIZE / 2 + { + let btree = mem::take(inner); + let vec = Vec::from_iter(btree); + mem::swap(self, &mut Self::Vec(vec)); + } + ret + } + + pub fn len(&self) -> usize { + match self { + Self::BTree(inner) => inner.len(), + Self::Vec(inner) => inner.len(), + } + } + + #[auto_enum(Iterator)] + pub fn values_mut(&mut self) -> impl Iterator { + match self { + Self::BTree(inner) => inner.values_mut(), + Self::Vec(inner) => inner.iter_mut().map(|(_, v)| v), + } + } +} diff --git a/src/stream/src/executor/join/mod.rs b/src/stream/src/executor/join/mod.rs new file mode 100644 index 0000000000000..b8bd5ff84d95f --- /dev/null +++ b/src/stream/src/executor/join/mod.rs @@ -0,0 +1,108 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +pub mod builder; +pub mod hash_join; +pub mod join_row_set; +pub mod row; + +/// The `JoinType` and `SideType` are to mimic a enum, because currently +/// enum is not supported in const generic. +// TODO: Use enum to replace this once [feature(adt_const_params)](https://github.com/rust-lang/rust/issues/95174) get completed. +pub type JoinTypePrimitive = u8; + +#[allow(non_snake_case, non_upper_case_globals)] +pub mod JoinType { + use super::JoinTypePrimitive; + pub const Inner: JoinTypePrimitive = 0; + pub const LeftOuter: JoinTypePrimitive = 1; + pub const RightOuter: JoinTypePrimitive = 2; + pub const FullOuter: JoinTypePrimitive = 3; + pub const LeftSemi: JoinTypePrimitive = 4; + pub const LeftAnti: JoinTypePrimitive = 5; + pub const RightSemi: JoinTypePrimitive = 6; + pub const RightAnti: JoinTypePrimitive = 7; +} + +pub type SideTypePrimitive = u8; +#[allow(non_snake_case, non_upper_case_globals)] +pub mod SideType { + use super::SideTypePrimitive; + pub const Left: SideTypePrimitive = 0; + pub const Right: SideTypePrimitive = 1; +} + +pub const fn is_outer_side(join_type: JoinTypePrimitive, side_type: SideTypePrimitive) -> bool { + join_type == JoinType::FullOuter + || (join_type == JoinType::LeftOuter && side_type == SideType::Left) + || (join_type == JoinType::RightOuter && side_type == SideType::Right) +} + +pub const fn outer_side_null(join_type: JoinTypePrimitive, side_type: SideTypePrimitive) -> bool { + join_type == JoinType::FullOuter + || (join_type == JoinType::LeftOuter && side_type == SideType::Right) + || (join_type == JoinType::RightOuter && side_type == SideType::Left) +} + +/// Send the update only once if the join type is semi/anti and the update is the same side as the +/// join +pub const fn forward_exactly_once( + join_type: JoinTypePrimitive, + side_type: SideTypePrimitive, +) -> bool { + ((join_type == JoinType::LeftSemi || join_type == JoinType::LeftAnti) + && side_type == SideType::Left) + || ((join_type == JoinType::RightSemi || join_type == JoinType::RightAnti) + && side_type == SideType::Right) +} + +pub const fn only_forward_matched_side( + join_type: JoinTypePrimitive, + side_type: SideTypePrimitive, +) -> bool { + ((join_type == JoinType::LeftSemi || join_type == JoinType::LeftAnti) + && side_type == SideType::Right) + || ((join_type == JoinType::RightSemi || join_type == JoinType::RightAnti) + && side_type == SideType::Left) +} + +pub const fn is_semi(join_type: JoinTypePrimitive) -> bool { + join_type == JoinType::LeftSemi || join_type == JoinType::RightSemi +} + +pub const fn is_anti(join_type: JoinTypePrimitive) -> bool { + join_type == JoinType::LeftAnti || join_type == JoinType::RightAnti +} + +pub const fn is_left_semi_or_anti(join_type: JoinTypePrimitive) -> bool { + join_type == JoinType::LeftSemi || join_type == JoinType::LeftAnti +} + +pub const fn is_right_semi_or_anti(join_type: JoinTypePrimitive) -> bool { + join_type == JoinType::RightSemi || join_type == JoinType::RightAnti +} + +pub const fn need_left_degree(join_type: JoinTypePrimitive) -> bool { + join_type == JoinType::FullOuter + || join_type == JoinType::LeftOuter + || join_type == JoinType::LeftAnti + || join_type == JoinType::LeftSemi +} + +pub const fn need_right_degree(join_type: JoinTypePrimitive) -> bool { + join_type == JoinType::FullOuter + || join_type == JoinType::RightOuter + || join_type == JoinType::RightAnti + || join_type == JoinType::RightSemi +} diff --git a/src/stream/src/executor/join/row.rs b/src/stream/src/executor/join/row.rs new file mode 100644 index 0000000000000..9ab133fc314ba --- /dev/null +++ b/src/stream/src/executor/join/row.rs @@ -0,0 +1,82 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_common::estimate_size::EstimateSize; +use risingwave_common::row::{self, CompactedRow, OwnedRow, Row, RowExt}; +use risingwave_common::types::{DataType, ScalarImpl}; + +use crate::executor::StreamExecutorResult; + +/// This is a row with a match degree +#[derive(Clone, Debug)] +pub struct JoinRow { + pub row: R, + pub degree: DegreeType, +} + +impl JoinRow { + pub fn new(row: R, degree: DegreeType) -> Self { + Self { row, degree } + } + + pub fn is_zero_degree(&self) -> bool { + self.degree == 0 + } + + /// Return row and degree in `Row` format. The degree part will be inserted in degree table + /// later, so a pk prefix will be added. + /// + /// * `state_order_key_indices` - the order key of `row` + pub fn to_table_rows<'a>( + &'a self, + state_order_key_indices: &'a [usize], + ) -> (&'a R, impl Row + 'a) { + let order_key = (&self.row).project(state_order_key_indices); + let degree = build_degree_row(order_key, self.degree); + (&self.row, degree) + } + + pub fn encode(&self) -> EncodedJoinRow { + EncodedJoinRow { + compacted_row: (&self.row).into(), + degree: self.degree, + } + } +} + +pub type DegreeType = u64; + +fn build_degree_row(order_key: impl Row, degree: DegreeType) -> impl Row { + order_key.chain(row::once(Some(ScalarImpl::Int64(degree as i64)))) +} + +#[derive(Clone, Debug, EstimateSize)] +pub struct EncodedJoinRow { + pub compacted_row: CompactedRow, + pub degree: DegreeType, +} + +impl EncodedJoinRow { + pub fn decode(&self, data_types: &[DataType]) -> StreamExecutorResult> { + Ok(JoinRow { + row: self.decode_row(data_types)?, + degree: self.degree, + }) + } + + fn decode_row(&self, data_types: &[DataType]) -> StreamExecutorResult { + let row = self.compacted_row.deserialize(data_types)?; + Ok(row) + } +} diff --git a/src/stream/src/executor/lookup.rs b/src/stream/src/executor/lookup.rs index 87cc163ea3dff..2c1de3170a801 100644 --- a/src/stream/src/executor/lookup.rs +++ b/src/stream/src/executor/lookup.rs @@ -14,11 +14,10 @@ use async_trait::async_trait; use futures::StreamExt; -use risingwave_common::catalog::Schema; use risingwave_common::types::DataType; use risingwave_storage::StateStore; -use crate::executor::{Barrier, BoxedMessageStream, Executor, PkIndicesRef}; +use crate::executor::{Barrier, BoxedMessageStream, Execute}; mod cache; mod sides; @@ -28,7 +27,7 @@ mod impl_; pub use impl_::LookupExecutorParams; -use super::{ActorContextRef, ExecutorInfo}; +use super::{ActorContextRef, Executor}; #[cfg(test)] mod tests; @@ -42,8 +41,6 @@ mod tests; pub struct LookupExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - /// the data types of the produced data chunk inside lookup (before reordering) chunk_data_types: Vec, @@ -54,10 +51,10 @@ pub struct LookupExecutor { stream: StreamJoinSide, /// The executor for arrangement. - arrangement_executor: Option>, + arrangement_executor: Option, /// The executor for stream. - stream_executor: Option>, + stream_executor: Option, /// The last received barrier. last_barrier: Option, @@ -83,20 +80,8 @@ pub struct LookupExecutor { } #[async_trait] -impl Executor for LookupExecutor { +impl Execute for LookupExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } diff --git a/src/stream/src/executor/lookup/impl_.rs b/src/stream/src/executor/lookup/impl_.rs index 8b86cfc602f08..db2511e19e7bd 100644 --- a/src/stream/src/executor/lookup/impl_.rs +++ b/src/stream/src/executor/lookup/impl_.rs @@ -31,8 +31,8 @@ use risingwave_storage::StateStore; use super::sides::{stream_lookup_arrange_prev_epoch, stream_lookup_arrange_this_epoch}; use crate::cache::cache_may_stale; use crate::common::metrics::MetricsInfo; -use crate::common::JoinStreamChunkBuilder; use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; +use crate::executor::join::builder::JoinStreamChunkBuilder; use crate::executor::lookup::cache::LookupCache; use crate::executor::lookup::sides::{ArrangeJoinSide, ArrangeMessage, StreamJoinSide}; use crate::executor::lookup::LookupExecutor; @@ -46,11 +46,11 @@ pub struct LookupExecutorParams { /// The side for arrangement. Currently, it should be a /// `MaterializeExecutor`. - pub arrangement: Box, + pub arrangement: Executor, /// The side for stream. It can be any stream, but it will generally be a /// `MaterializeExecutor`. - pub stream: Box, + pub stream: Executor, /// Should be the same as [`ColumnDesc`] in the arrangement. /// @@ -198,7 +198,6 @@ impl LookupExecutor { Self { ctx, - info, chunk_data_types, last_barrier: None, stream_executor: Some(stream), diff --git a/src/stream/src/executor/lookup/sides.rs b/src/stream/src/executor/lookup/sides.rs index 4fe3515bf4aa8..2e6a9803454f8 100644 --- a/src/stream/src/executor/lookup/sides.rs +++ b/src/stream/src/executor/lookup/sides.rs @@ -211,10 +211,7 @@ pub async fn align_barrier(mut left: BoxedMessageStream, mut right: BoxedMessage /// * Barrier (prev = `[2`], current = `[3`]) /// * `[Msg`] Arrangement (batch) #[try_stream(ok = ArrangeMessage, error = StreamExecutorError)] -pub async fn stream_lookup_arrange_prev_epoch( - stream: Box, - arrangement: Box, -) { +pub async fn stream_lookup_arrange_prev_epoch(stream: Executor, arrangement: Executor) { let mut input = pin!(align_barrier(stream.execute(), arrangement.execute())); let mut arrange_buf = vec![]; let mut stream_side_end = false; @@ -295,10 +292,7 @@ pub async fn stream_lookup_arrange_prev_epoch( /// * `[Do`] lookup `a` in arrangement of epoch `[2`] (current epoch) /// * Barrier (prev = `[2`], current = `[3`]) #[try_stream(ok = ArrangeMessage, error = StreamExecutorError)] -pub async fn stream_lookup_arrange_this_epoch( - stream: Box, - arrangement: Box, -) { +pub async fn stream_lookup_arrange_this_epoch(stream: Executor, arrangement: Executor) { let mut input = pin!(align_barrier(stream.execute(), arrangement.execute())); let mut stream_buf = vec![]; let mut arrange_buf = vec![]; @@ -433,14 +427,16 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (mut tx_l, source_l) = MockSource::channel(schema.clone(), vec![1]); - let (tx_r, source_r) = MockSource::channel(schema, vec![1]); - - let mut stream = stream_lookup_arrange_this_epoch( - Box::new(source_l.stop_on_finish(false)), - Box::new(source_r.stop_on_finish(false)), - ) - .boxed(); + let (mut tx_l, source_l) = MockSource::channel(); + let source_l = source_l + .stop_on_finish(false) + .into_executor(schema.clone(), vec![1]); + let (tx_r, source_r) = MockSource::channel(); + let source_r = source_r + .stop_on_finish(false) + .into_executor(schema, vec![1]); + + let mut stream = stream_lookup_arrange_this_epoch(source_l, source_r).boxed(); // Simulate recovery test drop(tx_r); diff --git a/src/stream/src/executor/lookup/tests.rs b/src/stream/src/executor/lookup/tests.rs index ed514f43f1320..17ff64cedb424 100644 --- a/src/stream/src/executor/lookup/tests.rs +++ b/src/stream/src/executor/lookup/tests.rs @@ -31,8 +31,8 @@ use crate::executor::lookup::impl_::LookupExecutorParams; use crate::executor::lookup::LookupExecutor; use crate::executor::test_utils::*; use crate::executor::{ - ActorContext, Barrier, BoxedMessageStream, Executor, ExecutorInfo, MaterializeExecutor, - Message, PkIndices, + ActorContext, Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, + MaterializeExecutor, Message, PkIndices, }; fn arrangement_col_descs() -> Vec { @@ -69,10 +69,7 @@ fn arrangement_col_arrange_rules_join_key() -> Vec { /// | + | 2337 | 8 | 3 | /// | - | 2333 | 6 | 3 | /// | b | | | 3 -> 4 | -async fn create_arrangement( - table_id: TableId, - memory_state_store: MemoryStateStore, -) -> Box { +async fn create_arrangement(table_id: TableId, memory_state_store: MemoryStateStore) -> Executor { // Two columns of int32 type, the second column is arrange key. let columns = arrangement_col_descs(); @@ -102,30 +99,32 @@ async fn create_arrangement( .collect_vec(), ); - let source = MockSource::with_messages( - schema, - vec![0], - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + ]) + .into_executor(schema, vec![0]); - Box::new( + Executor::new( + ExecutorInfo { + schema: source.schema().clone(), + pk_indices: source.pk_indices().to_vec(), + identity: "MaterializeExecutor".to_string(), + }, MaterializeExecutor::for_test( - Box::new(source), + source, memory_state_store, table_id, arrangement_col_arrange_rules(), column_ids, - 1, Arc::new(AtomicU64::new(0)), ConflictBehavior::NoCheck, ) - .await, + .await + .boxed(), ) } @@ -140,7 +139,7 @@ async fn create_arrangement( /// | b | | | 2 -> 3 | /// | - | 6 | 1 | 3 | /// | b | | | 3 -> 4 | -fn create_source() -> Box { +fn create_source() -> Executor { let columns = vec![ ColumnDesc::new_atomic(DataType::Int64, "join_column", 1), ColumnDesc::new_atomic(DataType::Int64, "rowid_column", 2), @@ -164,19 +163,14 @@ fn create_source() -> Box { .collect_vec(), ); - let source = MockSource::with_messages( - schema, - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - ], - ); - - Box::new(source) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + ]) + .into_executor(schema, PkIndices::new()) } async fn next_msg(buffer: &mut Vec, executor: &mut BoxedMessageStream) { @@ -206,7 +200,7 @@ async fn test_lookup_this_epoch() { identity: "LookupExecutor".to_string(), }; let lookup_executor = Box::new(LookupExecutor::new(LookupExecutorParams { - ctx: ActorContext::create(0), + ctx: ActorContext::for_test(0), info, arrangement, stream, @@ -280,7 +274,7 @@ async fn test_lookup_last_epoch() { identity: "LookupExecutor".to_string(), }; let lookup_executor = Box::new(LookupExecutor::new(LookupExecutorParams { - ctx: ActorContext::create(0), + ctx: ActorContext::for_test(0), info, arrangement, stream, diff --git a/src/stream/src/executor/lookup_union.rs b/src/stream/src/executor/lookup_union.rs index c34e938ab11db..1a8cff3ba3aba 100644 --- a/src/stream/src/executor/lookup_union.rs +++ b/src/stream/src/executor/lookup_union.rs @@ -18,35 +18,28 @@ use futures::future::{join_all, select, Either}; use futures::{FutureExt, SinkExt, StreamExt}; use futures_async_stream::try_stream; use itertools::Itertools; -use risingwave_common::catalog::Schema; use super::error::StreamExecutorError; -use super::*; -use crate::executor::{BoxedMessageStream, ExecutorInfo}; +use super::{Barrier, BoxedMessageStream, Execute, Executor, Message}; /// Merges data from multiple inputs with order. If `order = [2, 1, 0]`, then /// it will first pipe data from the third input; after the third input gets a barrier, it will then /// pipe the second, and finally the first. In the future we could have more efficient /// implementation. pub struct LookupUnionExecutor { - info: ExecutorInfo, - inputs: Vec, + inputs: Vec, order: Vec, } impl std::fmt::Debug for LookupUnionExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("LookupUnionExecutor") - .field("schema", &self.info.schema) - .field("pk_indices", &self.info.pk_indices) - .finish() + f.debug_struct("LookupUnionExecutor").finish() } } impl LookupUnionExecutor { - pub fn new(info: ExecutorInfo, inputs: Vec, order: Vec) -> Self { + pub fn new(inputs: Vec, order: Vec) -> Self { Self { - info, inputs, order: order.iter().map(|x| *x as _).collect(), } @@ -54,22 +47,10 @@ impl LookupUnionExecutor { } #[async_trait] -impl Executor for LookupUnionExecutor { +impl Execute for LookupUnionExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } impl LookupUnionExecutor { @@ -155,52 +136,36 @@ mod tests { let schema = Schema { fields: vec![Field::unnamed(DataType::Int64)], }; - let source0 = MockSource::with_messages( - schema.clone(), - vec![0], - vec![ - Message::Chunk(StreamChunk::from_pretty("I\n + 1")), - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(StreamChunk::from_pretty("I\n + 2")), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(StreamChunk::from_pretty("I\n + 3")), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - ], - ) - .stop_on_finish(false); - let source1 = MockSource::with_messages( - schema.clone(), - vec![0], - vec![ - Message::Chunk(StreamChunk::from_pretty("I\n + 11")), - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(StreamChunk::from_pretty("I\n + 12")), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - ], - ) - .stop_on_finish(false); - let source2 = MockSource::with_messages( - schema, - vec![0], - vec![ - Message::Chunk(StreamChunk::from_pretty("I\n + 21")), - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(StreamChunk::from_pretty("I\n + 22")), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - ], - ) - .stop_on_finish(false); - - let executor = Box::new(LookupUnionExecutor::new( - ExecutorInfo { - schema: source0.schema().clone(), - pk_indices: vec![0], - identity: "LookupUnionExecutor".to_string(), - }, - vec![Box::new(source0), Box::new(source1), Box::new(source2)], - vec![2, 1, 0], - )) - .execute(); + let source0 = MockSource::with_messages(vec![ + Message::Chunk(StreamChunk::from_pretty("I\n + 1")), + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(StreamChunk::from_pretty("I\n + 2")), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(StreamChunk::from_pretty("I\n + 3")), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + ]) + .stop_on_finish(false) + .into_executor(schema.clone(), vec![0]); + let source1 = MockSource::with_messages(vec![ + Message::Chunk(StreamChunk::from_pretty("I\n + 11")), + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(StreamChunk::from_pretty("I\n + 12")), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + ]) + .stop_on_finish(false) + .into_executor(schema.clone(), vec![0]); + let source2 = MockSource::with_messages(vec![ + Message::Chunk(StreamChunk::from_pretty("I\n + 21")), + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(StreamChunk::from_pretty("I\n + 22")), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + ]) + .stop_on_finish(false) + .into_executor(schema, vec![0]); + + let executor = LookupUnionExecutor::new(vec![source0, source1, source2], vec![2, 1, 0]) + .boxed() + .execute(); let outputs: Vec<_> = executor.try_collect().await.unwrap(); assert_eq!( diff --git a/src/stream/src/executor/managed_state/join/mod.rs b/src/stream/src/executor/managed_state/join/mod.rs index ff20e5346fc34..a1ca7bcdd7891 100644 --- a/src/stream/src/executor/managed_state/join/mod.rs +++ b/src/stream/src/executor/managed_state/join/mod.rs @@ -20,7 +20,7 @@ use std::ops::{Bound, Deref, DerefMut}; use std::sync::Arc; use anyhow::Context; -use futures::future::try_join; +use futures::future::{join, try_join}; use futures::StreamExt; use futures_async_stream::for_await; pub(super) use join_entry_state::JoinEntryState; @@ -450,8 +450,20 @@ impl JoinHashMap { let (table_iter, degree_table_iter) = try_join(table_iter_fut, degree_table_iter_fut).await?; - #[for_await] - for (row, degree) in table_iter.zip(degree_table_iter) { + let mut pinned_table_iter = std::pin::pin!(table_iter); + let mut pinned_degree_table_iter = std::pin::pin!(degree_table_iter); + loop { + // Iterate on both iterators and ensure they have same size. Basically `zip_eq()`. + let (row, degree) = + join(pinned_table_iter.next(), pinned_degree_table_iter.next()).await; + let (row, degree) = match (row, degree) { + (None, None) => break, + (None, Some(_)) | (Some(_), None) => { + panic!("mismatched row and degree table of join key: {:?}", &key) + } + (Some(r), Some(d)) => (r, d), + }; + let row = row?; let degree_row = degree?; let pk1 = row.key(); diff --git a/src/stream/src/executor/merge.rs b/src/stream/src/executor/merge.rs index 6f62f5c015546..6ffc372c9b6bd 100644 --- a/src/stream/src/executor/merge.rs +++ b/src/stream/src/executor/merge.rs @@ -20,7 +20,6 @@ use anyhow::Context as _; use futures::stream::{FusedStream, FuturesUnordered, StreamFuture}; use futures::{pin_mut, Stream, StreamExt}; use futures_async_stream::try_stream; -use risingwave_common::catalog::Schema; use tokio::time::Instant; use super::error::StreamExecutorError; @@ -38,9 +37,6 @@ pub struct MergeExecutor { /// The context of the actor. actor_context: ActorContextRef, - /// Logical Operator Info - info: ExecutorInfo, - /// Upstream channels. upstreams: Vec, @@ -61,7 +57,6 @@ impl MergeExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, fragment_id: FragmentId, upstream_fragment_id: FragmentId, inputs: Vec, @@ -71,7 +66,6 @@ impl MergeExecutor { ) -> Self { Self { actor_context: ctx, - info, upstreams: inputs, fragment_id, upstream_fragment_id, @@ -81,17 +75,12 @@ impl MergeExecutor { } #[cfg(test)] - pub fn for_test(inputs: Vec, schema: Schema) -> Self { + pub fn for_test(inputs: Vec) -> Self { use super::exchange::input::LocalInput; use crate::executor::exchange::input::Input; Self::new( - ActorContext::create(114), - ExecutorInfo { - schema, - pk_indices: vec![], - identity: "MergeExecutor".to_string(), - }, + ActorContext::for_test(114), 514, 1919, inputs @@ -245,22 +234,10 @@ impl MergeExecutor { } } -impl Executor for MergeExecutor { +impl Execute for MergeExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } /// A stream for merging messages from multiple upstreams. @@ -465,7 +442,7 @@ mod tests { use super::*; use crate::executor::exchange::input::RemoteInput; use crate::executor::exchange::permit::channel_for_test; - use crate::executor::{Barrier, Executor, Mutation}; + use crate::executor::{Barrier, Execute, Mutation}; use crate::task::test_utils::helper_make_local_actor; fn build_test_chunk(epoch: u64) -> StreamChunk { @@ -484,7 +461,7 @@ mod tests { txs.push(tx); rxs.push(rx); } - let merger = MergeExecutor::for_test(rxs, Schema::default()); + let merger = MergeExecutor::for_test(rxs); let mut handles = Vec::with_capacity(CHANNEL_NUMBER); let epochs = (10..100u64).step_by(10).collect_vec(); @@ -559,8 +536,6 @@ mod tests { #[tokio::test] async fn test_configuration_change() { - let schema = Schema { fields: vec![] }; - let actor_id = 233; let (untouched, old, new) = (234, 235, 238); // upstream actors let ctx = Arc::new(SharedContext::for_test()); @@ -595,13 +570,8 @@ mod tests { .try_collect() .unwrap(); - let merge = MergeExecutor::new( - ActorContext::create(actor_id), - ExecutorInfo { - schema, - pk_indices: vec![], - identity: "MergeExecutor".to_string(), - }, + let mut merge = MergeExecutor::new( + ActorContext::for_test(actor_id), fragment_id, upstream_fragment_id, inputs, @@ -612,8 +582,6 @@ mod tests { .boxed() .execute(); - pin_mut!(merge); - // 2. Take downstream receivers. let txs = [untouched, old, new] .into_iter() diff --git a/src/stream/src/executor/mod.rs b/src/stream/src/executor/mod.rs index 96244cabf0b13..7ec645e3b0dc7 100644 --- a/src/stream/src/executor/mod.rs +++ b/src/stream/src/executor/mod.rs @@ -20,7 +20,6 @@ use await_tree::InstrumentAwait; use enum_as_inner::EnumAsInner; use futures::stream::BoxStream; use futures::{Stream, StreamExt}; -use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::StreamChunk; use risingwave_common::buffer::Bitmap; @@ -63,16 +62,16 @@ mod dedup; mod dispatch; pub mod dml; mod dynamic_filter; -mod error; +pub mod error; mod expand; mod filter; mod flow_control; mod hash_agg; pub mod hash_join; mod hop_window; +mod join; mod lookup; mod lookup_union; -mod managed_state; mod merge; mod mview; mod no_op; @@ -90,6 +89,7 @@ mod sort_buffer; pub mod source; mod stateless_simple_agg; mod stream_reader; +mod subscription; pub mod subtask; mod temporal_join; mod top_n; @@ -122,6 +122,7 @@ pub use flow_control::FlowControlExecutor; pub use hash_agg::HashAggExecutor; pub use hash_join::*; pub use hop_window::HopWindowExecutor; +pub use join::JoinType; pub use lookup::*; pub use lookup_union::LookupUnionExecutor; pub use merge::MergeExecutor; @@ -139,6 +140,7 @@ pub use sink::SinkExecutor; pub use sort::*; pub use source::*; pub use stateless_simple_agg::StatelessSimpleAggExecutor; +pub use subscription::SubscriptionExecutor; pub use temporal_join::*; pub use top_n::{ AppendOnlyGroupTopNExecutor, AppendOnlyTopNExecutor, GroupTopNExecutor, TopNExecutor, @@ -151,7 +153,6 @@ pub use wrapper::WrapperExecutor; use self::barrier_align::AlignedMessageStream; -pub type BoxedExecutor = Box; pub type MessageStreamItem = StreamExecutorResult; pub type BoxedMessageStream = BoxStream<'static, MessageStreamItem>; @@ -163,48 +164,27 @@ pub trait MessageStream = futures::Stream + Send; /// Static information of an executor. #[derive(Debug, Default, Clone)] pub struct ExecutorInfo { - /// See [`Executor::schema`]. + /// The schema of the OUTPUT of the executor. pub schema: Schema, - /// See [`Executor::pk_indices`]. + /// The primary key indices of the OUTPUT of the executor. + /// Schema is used by both OLAP and streaming, therefore + /// pk indices are maintained independently. pub pk_indices: PkIndices, - /// See [`Executor::identity`]. + /// Identity of the executor. pub identity: String, } -/// `Executor` supports handling of control messages. -pub trait Executor: Send + 'static { +/// [`Execute`] describes the methods an executor should implement to handle control messages. +pub trait Execute: Send + 'static { fn execute(self: Box) -> BoxedMessageStream; - /// Return the schema of the OUTPUT of the executor. - fn schema(&self) -> &Schema; - - /// Return the primary key indices of the OUTPUT of the executor. - /// Schema is used by both OLAP and streaming, therefore - /// pk indices are maintained independently. - fn pk_indices(&self) -> PkIndicesRef<'_>; - - /// Identity of the executor. - fn identity(&self) -> &str; - fn execute_with_epoch(self: Box, _epoch: u64) -> BoxedMessageStream { self.execute() } - #[inline(always)] - fn info(&self) -> ExecutorInfo { - let schema = self.schema().to_owned(); - let pk_indices = self.pk_indices().to_owned(); - let identity = self.identity().to_owned(); - ExecutorInfo { - schema, - pk_indices, - identity, - } - } - - fn boxed(self) -> BoxedExecutor + fn boxed(self) -> Box where Self: Sized + Send + 'static, { @@ -212,12 +192,64 @@ pub trait Executor: Send + 'static { } } -impl std::fmt::Debug for BoxedExecutor { +/// [`Executor`] combines the static information ([`ExecutorInfo`]) and the executable object to +/// handle messages ([`Execute`]). +pub struct Executor { + info: ExecutorInfo, + execute: Box, +} + +impl Executor { + pub fn new(info: ExecutorInfo, execute: Box) -> Self { + Self { info, execute } + } + + pub fn info(&self) -> &ExecutorInfo { + &self.info + } + + pub fn schema(&self) -> &Schema { + &self.info.schema + } + + pub fn pk_indices(&self) -> PkIndicesRef<'_> { + &self.info.pk_indices + } + + pub fn identity(&self) -> &str { + &self.info.identity + } + + pub fn execute(self) -> BoxedMessageStream { + self.execute.execute() + } + + pub fn execute_with_epoch(self, epoch: u64) -> BoxedMessageStream { + self.execute.execute_with_epoch(epoch) + } +} + +impl std::fmt::Debug for Executor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.write_str(self.identity()) } } +impl From<(ExecutorInfo, Box)> for Executor { + fn from((info, execute): (ExecutorInfo, Box)) -> Self { + Self::new(info, execute) + } +} + +impl From<(ExecutorInfo, E)> for Executor +where + E: Execute, +{ + fn from((info, execute): (ExecutorInfo, E)) -> Self { + Self::new(info, execute.boxed()) + } +} + pub const INVALID_EPOCH: u64 = 0; type UpstreamFragmentId = FragmentId; diff --git a/src/stream/src/executor/monitor/streaming_stats.rs b/src/stream/src/executor/monitor/streaming_stats.rs index c21c0294a70dd..58abaa5e3bdc8 100644 --- a/src/stream/src/executor/monitor/streaming_stats.rs +++ b/src/stream/src/executor/monitor/streaming_stats.rs @@ -64,7 +64,6 @@ pub struct StreamingMetrics { // Source pub source_output_row_count: GenericCounterVec, - pub source_row_per_barrier: GenericCounterVec, pub source_split_change_count: GenericCounterVec, // Sink & materialized view @@ -179,6 +178,8 @@ pub struct StreamingMetrics { pub lru_evicted_watermark_time_ms: LabelGuardedIntGaugeVec<3>, pub jemalloc_allocated_bytes: IntGauge, pub jemalloc_active_bytes: IntGauge, + pub jemalloc_resident_bytes: IntGauge, + pub jemalloc_metadata_bytes: IntGauge, pub jvm_allocated_bytes: IntGauge, pub jvm_active_bytes: IntGauge, @@ -211,15 +212,7 @@ impl StreamingMetrics { let source_output_row_count = register_int_counter_vec_with_registry!( "stream_source_output_rows_counts", "Total number of rows that have been output from source", - &["source_id", "source_name", "actor_id"], - registry - ) - .unwrap(); - - let source_row_per_barrier = register_int_counter_vec_with_registry!( - "stream_source_rows_per_barrier_counts", - "Total number of rows that have been output from source per barrier", - &["actor_id", "executor_id"], + &["source_id", "source_name", "actor_id", "fragment_id"], registry ) .unwrap(); @@ -227,7 +220,7 @@ impl StreamingMetrics { let source_split_change_count = register_int_counter_vec_with_registry!( "stream_source_split_change_event_count", "Total number of split change events that have been operated by source", - &["source_id", "source_name", "actor_id"], + &["source_id", "source_name", "actor_id", "fragment_id"], registry ) .unwrap(); @@ -965,6 +958,20 @@ impl StreamingMetrics { ) .unwrap(); + let jemalloc_resident_bytes = register_int_gauge_with_registry!( + "jemalloc_resident_bytes", + "The active memory jemalloc, got from jemalloc_ctl", + registry + ) + .unwrap(); + + let jemalloc_metadata_bytes = register_int_gauge_with_registry!( + "jemalloc_metadata_bytes", + "The active memory jemalloc, got from jemalloc_ctl", + registry + ) + .unwrap(); + let jvm_allocated_bytes = register_int_gauge_with_registry!( "jvm_allocated_bytes", "The allocated jvm memory", @@ -1061,7 +1068,6 @@ impl StreamingMetrics { actor_in_record_cnt, actor_out_record_cnt, source_output_row_count, - source_row_per_barrier, source_split_change_count, sink_input_row_count, mview_input_row_count, @@ -1139,6 +1145,8 @@ impl StreamingMetrics { lru_evicted_watermark_time_ms, jemalloc_allocated_bytes, jemalloc_active_bytes, + jemalloc_resident_bytes, + jemalloc_metadata_bytes, jvm_allocated_bytes, jvm_active_bytes, materialize_cache_hit_count, diff --git a/src/stream/src/executor/mview/materialize.rs b/src/stream/src/executor/mview/materialize.rs index 539039f041e17..96fb1c2fc2fc9 100644 --- a/src/stream/src/executor/mview/materialize.rs +++ b/src/stream/src/executor/mview/materialize.rs @@ -42,15 +42,16 @@ use crate::common::table::state_table::StateTableInner; use crate::executor::error::StreamExecutorError; use crate::executor::monitor::StreamingMetrics; use crate::executor::{ - expect_first_barrier, ActorContext, ActorContextRef, BoxedExecutor, BoxedMessageStream, - Executor, ExecutorInfo, Message, PkIndicesRef, StreamExecutorResult, + expect_first_barrier, ActorContext, ActorContextRef, AddMutation, BoxedMessageStream, Execute, + Executor, Message, Mutation, StreamExecutorResult, UpdateMutation, }; -use crate::task::AtomicU64Ref; +use crate::task::{ActorId, AtomicU64Ref}; /// `MaterializeExecutor` materializes changes in stream into a materialized view on storage. pub struct MaterializeExecutor { - input: BoxedExecutor, - info: ExecutorInfo, + input: Executor, + + schema: Schema, state_table: StateTableInner, @@ -70,8 +71,8 @@ impl MaterializeExecutor { /// should be `None`. #[allow(clippy::too_many_arguments)] pub async fn new( - input: BoxedExecutor, - info: ExecutorInfo, + input: Executor, + schema: Schema, store: S, arrange_key: Vec, actor_context: ActorContextRef, @@ -83,14 +84,22 @@ impl MaterializeExecutor { ) -> Self { let arrange_key_indices: Vec = arrange_key.iter().map(|k| k.column_index).collect(); - let state_table = StateTableInner::from_table_catalog(table_catalog, store, vnodes).await; + // Note: The current implementation could potentially trigger a switch on the inconsistent_op flag. If the storage relies on this flag to perform optimizations, it would be advisable to maintain consistency with it throughout the lifecycle. + let state_table = if matches!(conflict_behavior, ConflictBehavior::Overwrite) + && actor_context.dispatch_num == 0 + { + // Table with overwrite conflict behavior could disable conflict check if no downstream mv depends on it, so we use a inconsistent_op to skip sanity check as well. + StateTableInner::from_table_catalog_inconsistent_op(table_catalog, store, vnodes).await + } else { + StateTableInner::from_table_catalog(table_catalog, store, vnodes).await + }; let metrics_info = MetricsInfo::new(metrics, table_catalog.id, actor_context.id, "Materialize"); Self { input, - info, + schema, state_table, arrange_key_indices, actor_context, @@ -106,7 +115,7 @@ impl MaterializeExecutor { let actor_id_str = self.actor_context.id.to_string(); let fragment_id_str = self.actor_context.fragment_id.to_string(); - let data_types = self.schema().data_types().clone(); + let data_types = self.schema.data_types(); let mut input = self.input.execute(); let barrier = expect_first_barrier(&mut input).await?; @@ -130,7 +139,9 @@ impl MaterializeExecutor { .inc_by(chunk.cardinality() as u64); match self.conflict_behavior { - ConflictBehavior::Overwrite | ConflictBehavior::IgnoreConflict => { + ConflictBehavior::Overwrite | ConflictBehavior::IgnoreConflict + if self.state_table.is_consistent_op() => + { if chunk.cardinality() == 0 { // empty chunk continue; @@ -181,8 +192,8 @@ impl MaterializeExecutor { None => continue, } } - - ConflictBehavior::NoCheck => { + ConflictBehavior::IgnoreConflict => unreachable!(), + ConflictBehavior::NoCheck | ConflictBehavior::Overwrite => { self.state_table.write_chunk(chunk.clone()); self.state_table.try_flush().await?; Message::Chunk(chunk) @@ -190,7 +201,18 @@ impl MaterializeExecutor { } } Message::Barrier(b) => { - self.state_table.commit(b.epoch).await?; + let mutation = b.mutation.clone(); + // If a downstream mv depends on the current table, we need to do conflict check again. + if !self.state_table.is_consistent_op() + && Self::new_downstream_created(mutation, self.actor_context.id) + { + assert_eq!(self.conflict_behavior, ConflictBehavior::Overwrite); + self.state_table + .commit_with_switch_consistent_op(b.epoch, Some(true)) + .await?; + } else { + self.state_table.commit(b.epoch).await?; + } // Update the vnode bitmap for the state table if asked. if let Some(vnode_bitmap) = b.as_update_vnode_bitmap(self.actor_context.id) { @@ -207,18 +229,46 @@ impl MaterializeExecutor { } } } + + fn new_downstream_created(mutation: Option>, actor_id: ActorId) -> bool { + let Some(mutation) = mutation.as_deref() else { + return false; + }; + match mutation { + // Add is for mv, index and sink creation. + Mutation::Add(AddMutation { adds, .. }) => adds.get(&actor_id).is_some(), + // AddAndUpdate is for sink-into-table. + Mutation::AddAndUpdate( + AddMutation { adds, .. }, + UpdateMutation { + dispatchers, + actor_new_dispatchers: actor_dispatchers, + .. + }, + ) => { + adds.get(&actor_id).is_some() + || actor_dispatchers.get(&actor_id).is_some() + || dispatchers.get(&actor_id).is_some() + } + Mutation::Update(_) + | Mutation::Stop(_) + | Mutation::Pause + | Mutation::Resume + | Mutation::SourceChangeSplit(_) + | Mutation::Throttle(_) => false, + } + } } impl MaterializeExecutor { /// Create a new `MaterializeExecutor` without distribution info for test purpose. #[allow(clippy::too_many_arguments)] pub async fn for_test( - input: BoxedExecutor, + input: Executor, store: S, table_id: TableId, keys: Vec, column_ids: Vec, - executor_id: u64, watermark_epoch: AtomicU64Ref, conflict_behavior: ConflictBehavior, ) -> Self { @@ -242,14 +292,10 @@ impl MaterializeExecutor { Self { input, + schema, state_table, arrange_key_indices: arrange_columns.clone(), - actor_context: ActorContext::create(0), - info: ExecutorInfo { - schema, - pk_indices: arrange_columns, - identity: format!("MaterializeExecutor {:X}", executor_id), - }, + actor_context: ActorContext::for_test(0), materialize_cache: MaterializeCache::new(watermark_epoch, MetricsInfo::for_test()), conflict_behavior, } @@ -378,32 +424,15 @@ impl MaterializeBuffer { self.buffer } } -impl Executor for MaterializeExecutor { +impl Execute for MaterializeExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> ExecutorInfo { - self.info.clone() - } } impl std::fmt::Debug for MaterializeExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("MaterializeExecutor") - .field("info", &self.info()) .field("arrange_key_indices", &self.arrange_key_indices) .finish() } @@ -626,17 +655,14 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + ]) + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -653,19 +679,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::NoCheck, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::NoCheck, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -734,17 +758,14 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + ]) + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -761,19 +782,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::Overwrite, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::Overwrite, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -830,18 +849,15 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(chunk1), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(chunk1), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + ]) + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -858,19 +874,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::Overwrite, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::Overwrite, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -962,19 +976,16 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + ]) + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -991,19 +1002,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::Overwrite, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::Overwrite, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -1146,18 +1155,15 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(chunk1), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(chunk1), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + ]) + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1174,19 +1180,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::IgnoreConflict, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::IgnoreConflict, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -1257,15 +1261,12 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + ]) + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1282,19 +1283,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::IgnoreConflict, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::IgnoreConflict, ) + .await + .boxed() .execute(); let _msg1 = materialize_executor .next() @@ -1376,19 +1375,16 @@ mod tests { ); // Prepare stream executors. - let source = MockSource::with_messages( - schema.clone(), - PkIndices::new(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(chunk1), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(chunk2), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(chunk3), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(chunk1), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(chunk2), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(chunk3), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + ]) + .into_executor(schema.clone(), PkIndices::new()); let order_types = vec![OrderType::ascending()]; let column_descs = vec![ @@ -1405,19 +1401,17 @@ mod tests { vec![0, 1], ); - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store, - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - ConflictBehavior::IgnoreConflict, - ) - .await, + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store, + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + ConflictBehavior::IgnoreConflict, ) + .await + .boxed() .execute(); materialize_executor.next().await.transpose().unwrap(); @@ -1587,21 +1581,20 @@ mod tests { )))) .collect(); // Prepare stream executors. - let source = MockSource::with_messages(schema.clone(), PkIndices::new(), messages); - - let mut materialize_executor = Box::new( - MaterializeExecutor::for_test( - Box::new(source), - memory_state_store.clone(), - table_id, - vec![ColumnOrder::new(0, OrderType::ascending())], - column_ids, - 1, - Arc::new(AtomicU64::new(0)), - conflict_behavior, - ) - .await, + let source = + MockSource::with_messages(messages).into_executor(schema.clone(), PkIndices::new()); + + let mut materialize_executor = MaterializeExecutor::for_test( + source, + memory_state_store.clone(), + table_id, + vec![ColumnOrder::new(0, OrderType::ascending())], + column_ids, + Arc::new(AtomicU64::new(0)), + conflict_behavior, ) + .await + .boxed() .execute(); materialize_executor.expect_barrier().await; diff --git a/src/stream/src/executor/no_op.rs b/src/stream/src/executor/no_op.rs index f116eced8864d..ac12bf99d5d7f 100644 --- a/src/stream/src/executor/no_op.rs +++ b/src/stream/src/executor/no_op.rs @@ -12,44 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. -use risingwave_common::catalog::Schema; - -use super::{ - ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, ExecutorInfo, PkIndicesRef, -}; +use super::{ActorContextRef, BoxedMessageStream, Execute, Executor}; /// No-op executor directly forwards the input stream. Currently used to break the multiple edges in /// the fragment graph. pub struct NoOpExecutor { _ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, } impl NoOpExecutor { - pub fn new(ctx: ActorContextRef, info: ExecutorInfo, input: BoxedExecutor) -> Self { - Self { - _ctx: ctx, - info, - input, - } + pub fn new(ctx: ActorContextRef, input: Executor) -> Self { + Self { _ctx: ctx, input } } } -impl Executor for NoOpExecutor { +impl Execute for NoOpExecutor { fn execute(self: Box) -> BoxedMessageStream { self.input.execute() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } diff --git a/src/stream/src/executor/now.rs b/src/stream/src/executor/now.rs index 853411089eaba..755e48f325965 100644 --- a/src/stream/src/executor/now.rs +++ b/src/stream/src/executor/now.rs @@ -18,7 +18,6 @@ use std::ops::Bound::Unbounded; use futures::{pin_mut, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; use risingwave_common::row::{self, OwnedRow}; use risingwave_common::types::{DataType, Datum}; use risingwave_storage::StateStore; @@ -26,13 +25,12 @@ use tokio::sync::mpsc::UnboundedReceiver; use tokio_stream::wrappers::UnboundedReceiverStream; use super::{ - Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, Mutation, PkIndicesRef, - StreamExecutorError, Watermark, + Barrier, BoxedMessageStream, Execute, Message, Mutation, StreamExecutorError, Watermark, }; use crate::common::table::state_table::StateTable; pub struct NowExecutor { - info: ExecutorInfo, + data_types: Vec, /// Receiver of barrier channel. barrier_receiver: UnboundedReceiver, @@ -42,24 +40,23 @@ pub struct NowExecutor { impl NowExecutor { pub fn new( - info: ExecutorInfo, + data_types: Vec, barrier_receiver: UnboundedReceiver, state_table: StateTable, ) -> Self { Self { - info, + data_types, barrier_receiver, state_table, } } #[try_stream(ok = Message, error = StreamExecutorError)] - async fn into_stream(self) { + async fn execute_inner(self) { let Self { + data_types, barrier_receiver, mut state_table, - info, - .. } = self; // Whether the executor is paused. @@ -102,9 +99,6 @@ impl NowExecutor { last_timestamp = state_row.and_then(|row| row[0].clone()); paused = barrier.is_pause_on_startup(); initialized = true; - } else if paused { - // Assert that no data is updated. - state_table.commit_no_data_expected(barrier.epoch); } else { state_table.commit(barrier.epoch).await?; } @@ -134,15 +128,12 @@ impl NowExecutor { let row = row::once(×tamp); state_table.update(last_row, row); - StreamChunk::from_rows( - &[(Op::Delete, last_row), (Op::Insert, row)], - &info.schema.data_types(), - ) + StreamChunk::from_rows(&[(Op::Delete, last_row), (Op::Insert, row)], &data_types) } else { let row = row::once(×tamp); state_table.insert(row); - StreamChunk::from_rows(&[(Op::Insert, row)], &info.schema.data_types()) + StreamChunk::from_rows(&[(Op::Insert, row)], &data_types) }; yield Message::Chunk(stream_chunk); @@ -158,28 +149,16 @@ impl NowExecutor { } } -impl Executor for NowExecutor { +impl Execute for NowExecutor { fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity + self.execute_inner().boxed() } } #[cfg(test)] mod tests { use risingwave_common::array::StreamChunk; - use risingwave_common::catalog::{ColumnDesc, ColumnId, Field, Schema, TableId}; + use risingwave_common::catalog::{ColumnDesc, ColumnId, TableId}; use risingwave_common::test_prelude::StreamChunkTestExt; use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_storage::memory::MemoryStateStore; @@ -189,8 +168,7 @@ mod tests { use crate::common::table::state_table::StateTable; use crate::executor::test_utils::StreamExecutorTestExt; use crate::executor::{ - Barrier, BoxedMessageStream, Executor, ExecutorInfo, Mutation, StreamExecutorResult, - Watermark, + Barrier, BoxedMessageStream, Execute, Mutation, StreamExecutorResult, Watermark, }; #[tokio::test] @@ -413,22 +391,8 @@ mod tests { let (sender, barrier_receiver) = unbounded_channel(); - let schema = Schema::new(vec![Field { - data_type: DataType::Timestamptz, - name: String::from("now"), - sub_fields: vec![], - type_name: String::default(), - }]); - - let now_executor = NowExecutor::new( - ExecutorInfo { - schema, - pk_indices: vec![], - identity: "NowExecutor".to_string(), - }, - barrier_receiver, - state_table, - ); - (sender, Box::new(now_executor).execute()) + let now_executor = + NowExecutor::new(vec![DataType::Timestamptz], barrier_receiver, state_table); + (sender, now_executor.boxed().execute()) } } diff --git a/src/stream/src/executor/over_window/eowc.rs b/src/stream/src/executor/over_window/eowc.rs index 601a2f536aa00..32f1fafe9b8db 100644 --- a/src/stream/src/executor/over_window/eowc.rs +++ b/src/stream/src/executor/over_window/eowc.rs @@ -39,8 +39,8 @@ use crate::cache::{new_unbounded, ManagedLruCache}; use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, StreamExecutorResult, }; use crate::task::AtomicU64Ref; @@ -95,14 +95,14 @@ type PartitionCache = ManagedLruCache; // TODO(rc): us /// - `WindowState` should output agg result for `curr output row`. /// - Recover: iterate through state table, push rows to `WindowState`, ignore ready windows. pub struct EowcOverWindowExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } struct ExecutorInner { actor_ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, calls: Vec, input_pk_indices: Vec, partition_key_indices: Vec, @@ -117,30 +117,18 @@ struct ExecutionVars { _phantom: PhantomData, } -impl Executor for EowcOverWindowExecutor { +impl Execute for EowcOverWindowExecutor { fn execute(self: Box) -> BoxedMessageStream { self.executor_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } } pub struct EowcOverWindowExecutorArgs { pub actor_ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, + pub schema: Schema, pub calls: Vec, pub partition_key_indices: Vec, pub order_key_index: usize, @@ -150,13 +138,13 @@ pub struct EowcOverWindowExecutorArgs { impl EowcOverWindowExecutor { pub fn new(args: EowcOverWindowExecutorArgs) -> Self { - let input_info = args.input.info(); + let input_info = args.input.info().clone(); Self { input: args.input, inner: ExecutorInner { actor_ctx: args.actor_ctx, - info: args.info, + schema: args.schema, calls: args.calls, input_pk_indices: input_info.pk_indices, partition_key_indices: args.partition_key_indices, @@ -237,7 +225,7 @@ impl EowcOverWindowExecutor { vars: &mut ExecutionVars, chunk: StreamChunk, ) -> StreamExecutorResult> { - let mut builders = this.info.schema.create_array_builders(chunk.capacity()); // just an estimate + let mut builders = this.schema.create_array_builders(chunk.capacity()); // just an estimate // We assume that the input is sorted by order key. for record in chunk.records() { @@ -318,7 +306,7 @@ impl EowcOverWindowExecutor { for key in keys_to_evict { let order_key = memcmp_encoding::decode_row( &key.order_key, - &[this.info.schema[this.order_key_index].data_type()], + &[this.schema[this.order_key_index].data_type()], &[OrderType::ascending()], )?; let state_row_pk = (&partition_key).chain(order_key).chain(key.pk); diff --git a/src/stream/src/executor/over_window/frame_finder.rs b/src/stream/src/executor/over_window/frame_finder.rs new file mode 100644 index 0000000000000..3154284653f11 --- /dev/null +++ b/src/stream/src/executor/over_window/frame_finder.rs @@ -0,0 +1,1583 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Helper functions for finding affected ranges from over window range cache for a +//! given set of changes (delta). + +use std::ops::Bound; + +use delta_btree_map::DeltaBTreeMap; +use itertools::Itertools; +use risingwave_common::row::OwnedRow; +use risingwave_common::types::{Datum, Sentinelled, ToDatumRef}; +use risingwave_common::util::memcmp_encoding; +use risingwave_common::util::sort_util::cmp_datum; +use risingwave_expr::window_function::{FrameBound, RangeFrameBounds, RowsFrameBounds, StateKey}; + +use super::over_partition::CacheKey; + +// -------------------------- ↓ PUBLIC INTERFACE ↓ -------------------------- + +/// Merge several `ROWS` frames into one super frame. The returned super frame is +/// guaranteed to be *canonical*, which means that the `CURRENT ROW` is always +/// included in the returned frame. +pub(super) fn merge_rows_frames(rows_frames: &[&RowsFrameBounds]) -> RowsFrameBounds { + if rows_frames.is_empty() { + // When there's no `ROWS` frame, for simplicity, we don't return `None`. Instead, + // we return `ROWS BETWEEN CURRENT ROW AND CURRENT ROW`, which is the implicit + // frame for all selected upstream columns. + // + // For example, the following two queries are equivalent: + // + // ```sql + // SELECT a, b, sum(c) OVER (...) FROM t; + // SELECT + // first_value(a) OVER (... ROWS BETWEEN CURRENT ROW AND CURRENT ROW), + // first_value(b) OVER (... ROWS BETWEEN CURRENT ROW AND CURRENT ROW), + // sum(c) OVER (...) + // FROM t; + // ``` + // + // See https://risingwave.com/blog/risingwave-window-functions-the-art-of-sliding-and-the-aesthetics-of-symmetry/ + // for more details. + return RowsFrameBounds { + start: FrameBound::CurrentRow, + end: FrameBound::CurrentRow, + }; + } + + let none_as_max_cmp = |x: &Option, y: &Option| match (x, y) { + // None means unbounded, which should be the largest. + (None, None) => std::cmp::Ordering::Equal, + (None, Some(_)) => std::cmp::Ordering::Greater, + (Some(_), None) => std::cmp::Ordering::Less, + (Some(x), Some(y)) => x.cmp(y), + }; + + // Note that the following two both use `max_by`, unlike when handling `RANGE` frames, + // because here for `ROWS` frames, offsets are unsigned. We convert all pure preceding/ + // following `ROWS` frames into one containing the `CURRENT ROW`. + let start = rows_frames + .iter() + .map(|bounds| bounds.n_preceding_rows()) + .max_by(none_as_max_cmp) + .unwrap(); + let end = rows_frames + .iter() + .map(|bounds| bounds.n_following_rows()) + .max_by(none_as_max_cmp) + .unwrap(); + + RowsFrameBounds { + start: start + .map(FrameBound::Preceding) // may produce Preceding(0), but doesn't matter + .unwrap_or(FrameBound::UnboundedPreceding), + end: end + .map(FrameBound::Following) // may produce Following(0), but doesn't matter + .unwrap_or(FrameBound::UnboundedFollowing), + } +} + +/// For a canonical `ROWS` frame, given a key in delta, find the cache key +/// corresponding to the CURRENT ROW of the first frame that contains the given +/// key. +/// +/// ## Example +/// +/// - Frame: `ROWS BETWEEN ? AND 2 FOLLOWING` +/// - Cache: `[1, 2, 5]` +/// - Delta: `[Delete 5]` +/// +/// For delta key `5`, this function will return `1` as the *first curr key*. +/// +/// More examples can be found in the comment inside [`find_curr_for_rows_frame`]. +pub(super) fn find_first_curr_for_rows_frame<'cache>( + frame_bounds: &'_ RowsFrameBounds, + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + delta_key: &'cache CacheKey, +) -> &'cache CacheKey { + find_curr_for_rows_frame::(frame_bounds, part_with_delta, delta_key) +} + +/// For a canonical `ROWS` frame, given a key in delta, find the cache key +/// corresponding to the CURRENT ROW of the last frame that contains the given +/// key. +/// +/// This is the symmetric function of [`find_first_curr_for_rows_frame`]. +pub(super) fn find_last_curr_for_rows_frame<'cache>( + frame_bounds: &'_ RowsFrameBounds, + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + delta_key: &'cache CacheKey, +) -> &'cache CacheKey { + find_curr_for_rows_frame::(frame_bounds, part_with_delta, delta_key) +} + +/// For a canonical `ROWS` frame, given a key in `part_with_delta` corresponding +/// to some CURRENT ROW, find the cache key corresponding to the start row in +/// that frame. +pub(super) fn find_frame_start_for_rows_frame<'cache>( + frame_bounds: &'_ RowsFrameBounds, + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + curr_key: &'cache CacheKey, +) -> &'cache CacheKey { + find_boundary_for_rows_frame::(frame_bounds, part_with_delta, curr_key) +} + +/// For a canonical `ROWS` frame, given a key in `part_with_delta` corresponding +/// to some CURRENT ROW, find the cache key corresponding to the end row in that +/// frame. +/// +/// This is the symmetric function of [`find_frame_start_for_rows_frame`]. +pub(super) fn find_frame_end_for_rows_frame<'cache>( + frame_bounds: &'_ RowsFrameBounds, + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + curr_key: &'cache CacheKey, +) -> &'cache CacheKey { + find_boundary_for_rows_frame::(frame_bounds, part_with_delta, curr_key) +} + +/// Given the first and last key in delta, calculate the order values of the first +/// and the last frames logically affected by some `RANGE` frames. +pub(super) fn calc_logical_curr_for_range_frames( + range_frames: &[&RangeFrameBounds], + delta_first_key: &StateKey, + delta_last_key: &StateKey, +) -> Option<(Sentinelled, Sentinelled)> { + calc_logical_ord_for_range_frames( + range_frames, + delta_first_key, + delta_last_key, + |bounds, v| bounds.first_curr_of(v), + |bounds, v| bounds.last_curr_of(v), + ) +} + +/// Given the curr keys of the first and the last affected frames, calculate the order +/// values of the logical start row of the first frame and the logical end row of the +/// last frame. +pub(super) fn calc_logical_boundary_for_range_frames( + range_frames: &[&RangeFrameBounds], + first_curr_key: &StateKey, + last_curr_key: &StateKey, +) -> Option<(Sentinelled, Sentinelled)> { + calc_logical_ord_for_range_frames( + range_frames, + first_curr_key, + last_curr_key, + |bounds, v| bounds.frame_start_of(v), + |bounds, v| bounds.frame_end_of(v), + ) +} + +/// Given a left logical order value (e.g. first curr order value, first delta order value), +/// find the most closed cache key in `part_with_delta`. Ideally this function returns +/// the smallest key that is larger than or equal to the given logical order (using `lower_bound`). +pub(super) fn find_left_for_range_frames<'cache>( + range_frames: &[&RangeFrameBounds], + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + logical_order_value: impl ToDatumRef, + cache_key_pk_len: usize, // this is dirty but we have no better choice +) -> &'cache CacheKey { + find_for_range_frames::( + range_frames, + part_with_delta, + logical_order_value, + cache_key_pk_len, + ) +} + +/// Given a right logical order value (e.g. last curr order value, last delta order value), +/// find the most closed cache key in `part_with_delta`. Ideally this function returns +/// the largest key that is smaller than or equal to the given logical order (using `lower_bound`). +pub(super) fn find_right_for_range_frames<'cache>( + range_frames: &[&RangeFrameBounds], + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + logical_order_value: impl ToDatumRef, + cache_key_pk_len: usize, // this is dirty but we have no better choice +) -> &'cache CacheKey { + find_for_range_frames::( + range_frames, + part_with_delta, + logical_order_value, + cache_key_pk_len, + ) +} + +// -------------------------- ↑ PUBLIC INTERFACE ↑ -------------------------- + +fn find_curr_for_rows_frame<'cache, const LEFT: bool>( + frame_bounds: &'_ RowsFrameBounds, + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + delta_key: &'cache CacheKey, +) -> &'cache CacheKey { + debug_assert!(frame_bounds.is_canonical()); + if LEFT { + debug_assert!( + !frame_bounds.end.is_unbounded_following(), + "no need to call this function whenever any frame end is unbounded" + ); + } else { + debug_assert!( + !frame_bounds.start.is_unbounded_preceding(), + "no need to call this function whenever any frame start is unbounded" + ); + } + debug_assert!( + part_with_delta.first_key().is_some(), + "must have something in the range cache after applying delta" + ); + + // Let's think about the following algorithm with the following cases in mind. + // Insertions are relatively easy, so we only give the deletion examples. + // The two directions are symmetrical, we only consider the left direction in + // the following description. + // + // ## Background + // + // *Sm* means smallest sentinel node in the cache + // *SM* means largest sentinel node in the cache + // + // Before calling this function, all entries within range covered by delta should + // have been loaded into the cache. + // + // ## Cases + // + // Frame: ROWS BETWEEN ? AND 2 FOLLOWING + // Delta: Delete 5 + // Cache + Delta: + // [1, 2, 6] -> 1 + // [1, 6] -> 1 + // [6, 7, 8] -> 6, not precise but won't do too much harm + // [1, 2] -> 1 + // [1] -> 1 + // [Sm, 6] -> Sm + // [Sm, 1] -> Sm + // [1, 2, SM] -> 1 + // [1, SM] -> 1 + // [Sm, 1, SM] -> Sm + // [Sm, SM] -> Sm + // + // Frame: ROWS BETWEEN ? AND CURRENT ROW + // Delta: Delete 5 + // Cache + Delta: + // [1, 2, 6] -> 6, not precise but won't do too much harm + // [1, 2] -> 2, not precise but won't do too much harm + // [1, 2, SM] -> SM, not precise but won't do too much harm + // [Sm, SM] -> SM, not precise but won't do too much harm + // + // Frame: ROWS BETWEEN ? AND 2 PRECEDING + // This will be treated as if it's `ROWS BETWEEN ? AND CURRENT ROW`. + + let mut cursor = if LEFT { + part_with_delta.lower_bound(Bound::Included(delta_key)) + } else { + part_with_delta.upper_bound(Bound::Included(delta_key)) + }; + let n_rows_to_move = if LEFT { + frame_bounds.n_following_rows().unwrap() + } else { + frame_bounds.n_preceding_rows().unwrap() + }; + + if n_rows_to_move == 0 { + return cursor + .key() + .or_else(|| { + if LEFT { + part_with_delta.last_key() + } else { + part_with_delta.first_key() + } + }) + .unwrap(); + } + + for _ in 0..n_rows_to_move { + // Note that we have to move before check, to handle situation where the + // cursor is at ghost position at first. + if LEFT { + cursor.move_prev(); + } else { + cursor.move_next(); + } + if cursor.position().is_ghost() { + break; + } + } + cursor + .key() + .or_else(|| { + // Note the difference between this with the `n_rows_to_move == 0` case. + if LEFT { + part_with_delta.first_key() + } else { + part_with_delta.last_key() + } + }) + .unwrap() +} + +fn find_boundary_for_rows_frame<'cache, const LEFT: bool>( + frame_bounds: &'_ RowsFrameBounds, + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + curr_key: &'cache CacheKey, +) -> &'cache CacheKey { + debug_assert!(frame_bounds.is_canonical()); + if LEFT { + debug_assert!( + !frame_bounds.start.is_unbounded_preceding(), + "no need to call this function whenever any frame start is unbounded" + ); + } else { + debug_assert!( + !frame_bounds.end.is_unbounded_following(), + "no need to call this function whenever any frame end is unbounded" + ); + } + + // Now things are easier than in `find_curr_for_rows_frame`, because we already + // have `curr_key` which definitely exists in the `part_with_delta`. We just find + // the cursor pointing to it and move the cursor to frame boundary. + + let mut cursor = part_with_delta.find(curr_key).unwrap(); + assert!(!cursor.position().is_ghost()); + + let n_rows_to_move = if LEFT { + frame_bounds.n_preceding_rows().unwrap() + } else { + frame_bounds.n_following_rows().unwrap() + }; + + for _ in 0..n_rows_to_move { + if LEFT { + cursor.move_prev(); + } else { + cursor.move_next(); + } + if cursor.position().is_ghost() { + break; + } + } + cursor + .key() + .or_else(|| { + if LEFT { + part_with_delta.first_key() + } else { + part_with_delta.last_key() + } + }) + .unwrap() +} + +/// Given a pair of left and right state keys, calculate the leftmost (smallest) and rightmost +/// (largest) order values after the two given `offset_fn`s are applied, for all range frames. +/// +/// A more vivid description may be: Given a pair of left and right keys, this function pushes +/// the keys leftward and rightward respectively according to the given `offset_fn`s. +/// +/// This is not very understandable but we have to extract the code to a function to avoid +/// repeating. Check [`calc_logical_curr_for_range_frames`] and [`calc_logical_boundary_for_range_frames`] +/// if you cannot understand the purpose of this function. +fn calc_logical_ord_for_range_frames( + range_frames: &[&RangeFrameBounds], + left_key: &StateKey, + right_key: &StateKey, + left_offset_fn: impl Fn(&RangeFrameBounds, &Datum) -> Sentinelled, + right_offset_fn: impl Fn(&RangeFrameBounds, &Datum) -> Sentinelled, +) -> Option<(Sentinelled, Sentinelled)> { + if range_frames.is_empty() { + return None; + } + + let (data_type, order_type) = range_frames + .iter() + .map(|bounds| (&bounds.order_data_type, bounds.order_type)) + .all_equal_value() + .unwrap(); + + let datum_cmp = |a: &Datum, b: &Datum| cmp_datum(a, b, order_type); + + let left_given_ord = memcmp_encoding::decode_value(data_type, &left_key.order_key, order_type) + .expect("no reason to fail because we just encoded it in memory"); + let right_given_ord = + memcmp_encoding::decode_value(data_type, &right_key.order_key, order_type) + .expect("no reason to fail because we just encoded it in memory"); + + let logical_left_offset_ord = { + let mut order_value = None; + for bounds in range_frames { + let new_order_value = left_offset_fn(bounds, &left_given_ord); + order_value = match (order_value, new_order_value) { + (None, any_new) => Some(any_new), + (Some(old), new) => Some(std::cmp::min_by(old, new, |x, y| x.cmp_by(y, datum_cmp))), + }; + if !order_value.as_ref().unwrap().is_normal() { + // already unbounded + assert!( + order_value.as_ref().unwrap().is_smallest(), + "left order value should never be `Largest`" + ); + break; + } + } + order_value.expect("# of range frames > 0") + }; + + let logical_right_offset_ord = { + let mut order_value = None; + for bounds in range_frames { + let new_order_value = right_offset_fn(bounds, &right_given_ord); + order_value = match (order_value, new_order_value) { + (None, any_new) => Some(any_new), + (Some(old), new) => Some(std::cmp::max_by(old, new, |x, y| x.cmp_by(y, datum_cmp))), + }; + if !order_value.as_ref().unwrap().is_normal() { + // already unbounded + assert!( + order_value.as_ref().unwrap().is_largest(), + "right order value should never be `Smallest`" + ); + break; + } + } + order_value.expect("# of range frames > 0") + }; + + Some((logical_left_offset_ord, logical_right_offset_ord)) +} + +fn find_for_range_frames<'cache, const LEFT: bool>( + range_frames: &[&RangeFrameBounds], + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + logical_order_value: impl ToDatumRef, + cache_key_pk_len: usize, +) -> &'cache CacheKey { + debug_assert!( + part_with_delta.first_key().is_some(), + "must have something in the range cache after applying delta" + ); + + let order_type = range_frames + .iter() + .map(|bounds| bounds.order_type) + .all_equal_value() + .unwrap(); + + let search_key = Sentinelled::Normal(StateKey { + order_key: memcmp_encoding::encode_value(logical_order_value, order_type) + .expect("the data type is simple, should succeed"), + pk: if LEFT { + OwnedRow::empty() // empty row is minimal + } else { + OwnedRow::new(vec![None; cache_key_pk_len]) // all-NULL row is maximal in default order + } + .into(), + }); + + if LEFT { + let cursor = part_with_delta.lower_bound(Bound::Included(&search_key)); + if let Some((prev_key, _)) = cursor.peek_prev() + && prev_key.is_smallest() + { + // If the found lower bound of search key is right behind a smallest sentinel, + // we don't know if there's any other rows with the same order key in the state + // table but not in cache. We should conservatively return the sentinel key as + // the curr key. + prev_key + } else { + // If cursor is in ghost position, it simply means that the search key is larger + // than any existing key. Returning the last key in this case does no harm. Especially, + // if the last key is largest sentinel, the caller should extend the cache rightward + // to get possible entries with the same order value into the cache. + cursor.key().or_else(|| part_with_delta.last_key()).unwrap() + } + } else { + let cursor = part_with_delta.upper_bound(Bound::Included(&search_key)); + if let Some((next_key, _)) = cursor.peek_next() + && next_key.is_largest() + { + next_key + } else { + cursor + .key() + .or_else(|| part_with_delta.first_key()) + .unwrap() + } + } +} + +#[cfg(test)] +mod tests { + use std::collections::BTreeMap; + + use delta_btree_map::Change; + use risingwave_common::types::{ScalarImpl, Sentinelled}; + use risingwave_expr::window_function::FrameBound::*; + use risingwave_expr::window_function::{RowsFrameBounds, StateKey}; + + use super::*; + + #[test] + fn test_merge_rows_frame() { + fn assert_equivalent(bounds1: RowsFrameBounds, bounds2: RowsFrameBounds) { + assert_eq!(bounds1.start.to_offset(), bounds2.start.to_offset()); + assert_eq!(bounds1.end.to_offset(), bounds2.end.to_offset()); + } + + assert_equivalent( + merge_rows_frames(&[]), + RowsFrameBounds { + start: CurrentRow, + end: CurrentRow, + }, + ); + + let frames = [ + &RowsFrameBounds { + start: Preceding(3), + end: Preceding(2), + }, + &RowsFrameBounds { + start: Preceding(1), + end: Preceding(4), + }, + ]; + assert_equivalent( + merge_rows_frames(&frames), + RowsFrameBounds { + start: Preceding(4), + end: CurrentRow, + }, + ); + + let frames = [ + &RowsFrameBounds { + start: Preceding(3), + end: Following(2), + }, + &RowsFrameBounds { + start: Preceding(2), + end: Following(3), + }, + ]; + assert_equivalent( + merge_rows_frames(&frames), + RowsFrameBounds { + start: Preceding(3), + end: Following(3), + }, + ); + + let frames = [ + &RowsFrameBounds { + start: UnboundedPreceding, + end: Following(2), + }, + &RowsFrameBounds { + start: Preceding(2), + end: UnboundedFollowing, + }, + ]; + assert_equivalent( + merge_rows_frames(&frames), + RowsFrameBounds { + start: UnboundedPreceding, + end: UnboundedFollowing, + }, + ); + + let frames = [ + &RowsFrameBounds { + start: UnboundedPreceding, + end: Following(2), + }, + &RowsFrameBounds { + start: Following(5), + end: Following(2), + }, + ]; + assert_equivalent( + merge_rows_frames(&frames), + RowsFrameBounds { + start: UnboundedPreceding, + end: Following(5), + }, + ); + } + + macro_rules! create_cache { + (..., $( $pk:literal ),* , ...) => { + { + let mut cache = create_cache!( $( $pk ),* ); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + (..., $( $pk:literal ),*) => { + { + let mut cache = create_cache!( $( $pk ),* ); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache + } + }; + ($( $pk:literal ),* , ...) => { + { + let mut cache = create_cache!( $( $pk ),* ); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + ($( $pk:literal ),*) => { + { + #[allow(unused_mut)] + let mut cache = BTreeMap::new(); + $( + cache.insert( + CacheKey::Normal( + StateKey { + // order key doesn't matter here + order_key: vec![].into(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + OwnedRow::empty(), + ); + )* + cache + } + }; + ($ord_type:expr, [..., $( ( $ord:literal, $pk:literal ) ),* , ...]) => { + { + let mut cache = create_cache!($ord_type, [$( ( $ord, $pk ) ),*]); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + ($ord_type:expr, [..., $( ( $ord:literal, $pk:literal ) ),*]) => { + { + let mut cache = create_cache!($ord_type, [$( ( $ord, $pk ) ),*]); + cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); + cache + } + }; + ($ord_type:expr, [$( ( $ord:literal, $pk:literal ) ),* , ...]) => { + { + let mut cache = create_cache!($ord_type, [$( ( $ord, $pk ) ),*]); + cache.insert(CacheKey::Largest, OwnedRow::empty().into()); + cache + } + }; + ($ord_type:expr, [$( ( $ord:literal, $pk:literal ) ),*]) => { + { + #[allow(unused_mut)] + let mut cache = BTreeMap::new(); + $( + cache.insert( + CacheKey::Normal( + StateKey { + order_key: memcmp_encoding::encode_value( + Some(ScalarImpl::from($ord)), + $ord_type, + ).unwrap(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + OwnedRow::empty(), + ); + )* + cache + } + } + } + + macro_rules! create_change { + (Delete) => { + Change::Delete + }; + (Insert) => { + Change::Insert(OwnedRow::empty()) + }; + } + + macro_rules! create_delta { + ($( ( $pk:literal, $change:ident ) ),+ $(,)?) => { + { + let mut delta = BTreeMap::new(); + $( + delta.insert( + CacheKey::Normal( + StateKey { + // order key doesn't matter here + order_key: vec![].into(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + create_change!( $change ), + ); + )* + delta + } + }; + ($ord_type:expr, [ $( ( $ord:literal, $pk:literal, $change:ident ) ),+ $(,)? ]) => { + { + let mut delta = BTreeMap::new(); + $( + delta.insert( + CacheKey::Normal( + StateKey { + order_key: memcmp_encoding::encode_value( + Some(ScalarImpl::from($ord)), + $ord_type, + ).unwrap(), + pk: OwnedRow::new(vec![Some(ScalarImpl::from($pk))]).into(), + }, + ), + // value row doesn't matter here + create_change!( $change ), + ); + )* + delta + } + }; + } + + mod rows_frame_tests { + use super::*; + + fn assert_cache_key_eq(given: &CacheKey, expected: impl Into) { + assert_eq!( + given.as_normal_expect().pk.0, + OwnedRow::new(vec![Some(expected.into())]) + ) + } + + #[test] + fn test_insert_delta_only() { + let cache = create_cache!(); + let delta = create_delta!((1, Insert), (2, Insert), (3, Insert)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + let bounds = RowsFrameBounds { + start: Preceding(2), + end: CurrentRow, + }; + + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 1); + assert_cache_key_eq(last_curr_key, 3); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 1); + assert_cache_key_eq(last_frame_end, 3); + } + + #[test] + fn test_simple() { + let cache = create_cache!(1, 2, 3, 4, 5, 6); + let delta = create_delta!((2, Insert), (3, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + { + let bounds = RowsFrameBounds { + start: Preceding(2), + end: CurrentRow, + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 2); + assert_cache_key_eq(last_curr_key, 5); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 1); + assert_cache_key_eq(last_frame_end, 5); + } + + { + let bounds = RowsFrameBounds { + start: Preceding(1), + end: Following(2), + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 1); + assert_cache_key_eq(last_curr_key, 4); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 1); + assert_cache_key_eq(last_frame_end, 6); + } + + { + let bounds = RowsFrameBounds { + start: CurrentRow, + end: Following(2), + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 1); + assert_cache_key_eq(last_curr_key, 2); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 1); + assert_cache_key_eq(last_frame_end, 5); + } + } + + #[test] + fn test_lag_corner_case() { + let cache = create_cache!(1, 2, 3, 4, 5, 6); + let delta = create_delta!((1, Delete), (2, Delete), (3, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + let bounds = RowsFrameBounds { + start: Preceding(1), + end: CurrentRow, + }; + + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 4); + assert_cache_key_eq(last_curr_key, 4); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 4); + assert_cache_key_eq(last_frame_end, 4); + } + + #[test] + fn test_lead_corner_case() { + let cache = create_cache!(1, 2, 3, 4, 5, 6); + let delta = create_delta!((4, Delete), (5, Delete), (6, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + let bounds = RowsFrameBounds { + start: CurrentRow, + end: Following(1), + }; + + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 3); + assert_cache_key_eq(last_curr_key, 3); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 3); + assert_cache_key_eq(last_frame_end, 3); + } + + #[test] + fn test_lag_lead_offset_0_corner_case_1() { + let cache = create_cache!(1, 2, 3, 4); + let delta = create_delta!((2, Delete), (3, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + let bounds = RowsFrameBounds { + start: CurrentRow, + end: CurrentRow, + }; + + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 4); + assert_cache_key_eq(last_curr_key, 1); + + // first_curr_key > last_curr_key, should not continue to find frame start/end + } + + #[test] + fn test_lag_lead_offset_0_corner_case_2() { + // Note this case is false-positive, but it does very little harm. + + let cache = create_cache!(1, 2, 3, 4); + let delta = create_delta!((2, Delete), (3, Delete), (4, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + let bounds = RowsFrameBounds { + start: CurrentRow, + end: CurrentRow, + }; + + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 1); + assert_cache_key_eq(last_curr_key, 1); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 1); + assert_cache_key_eq(last_frame_end, 1); + } + + #[test] + fn test_lag_lead_offset_0_corner_case_3() { + let cache = create_cache!(1, 2, 3, 4, 5); + let delta = create_delta!((2, Delete), (3, Insert), (4, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + let bounds = RowsFrameBounds { + start: CurrentRow, + end: CurrentRow, + }; + + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 3); + assert_cache_key_eq(last_curr_key, 3); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 3); + assert_cache_key_eq(last_frame_end, 3); + } + + #[test] + fn test_empty_with_sentinels() { + let cache: BTreeMap, OwnedRow> = create_cache!(..., , ...); + let delta = create_delta!((1, Insert), (2, Insert)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + { + let bounds = RowsFrameBounds { + start: CurrentRow, + end: CurrentRow, + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 1); + assert_cache_key_eq(last_curr_key, 2); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 1); + assert_cache_key_eq(last_frame_end, 2); + } + + { + let bounds = RowsFrameBounds { + start: Preceding(1), + end: CurrentRow, + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 1); + assert!(last_curr_key.is_largest()); + + // reaches sentinel, should not continue to find frame start/end + } + + { + let bounds = RowsFrameBounds { + start: CurrentRow, + end: Following(3), + }; + + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert!(first_curr_key.is_smallest()); + assert_cache_key_eq(last_curr_key, 2); + + // reaches sentinel, should not continue to find frame start/end + } + } + + #[test] + fn test_with_left_sentinel() { + let cache = create_cache!(..., 2, 4, 5, 8); + let delta = create_delta!((3, Insert), (4, Insert), (8, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + { + let bounds = RowsFrameBounds { + start: CurrentRow, + end: Following(1), + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 2); + assert_cache_key_eq(last_curr_key, 5); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 2); + assert_cache_key_eq(last_frame_end, 5); + } + + { + let bounds = RowsFrameBounds { + start: Preceding(1), + end: Following(1), + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 2); + assert_cache_key_eq(last_curr_key, 5); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert!(first_frame_start.is_smallest()); + assert_cache_key_eq(last_frame_end, 5); + } + } + + #[test] + fn test_with_right_sentinel() { + let cache = create_cache!(1, 2, 4, 5, 8, ...); + let delta = create_delta!((3, Insert), (4, Insert), (5, Delete)); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + let delta_first_key = delta.first_key_value().unwrap().0; + let delta_last_key = delta.last_key_value().unwrap().0; + + { + let bounds = RowsFrameBounds { + start: Preceding(1), + end: CurrentRow, + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 3); + assert_cache_key_eq(last_curr_key, 8); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 2); + assert_cache_key_eq(last_frame_end, 8); + } + + { + let bounds = RowsFrameBounds { + start: Preceding(1), + end: Following(1), + }; + let first_curr_key = + find_first_curr_for_rows_frame(&bounds, part_with_delta, delta_first_key); + let last_curr_key = + find_last_curr_for_rows_frame(&bounds, part_with_delta, delta_last_key); + assert_cache_key_eq(first_curr_key, 2); + assert_cache_key_eq(last_curr_key, 8); + + let first_frame_start = + find_frame_start_for_rows_frame(&bounds, part_with_delta, first_curr_key); + let last_frame_end = + find_frame_end_for_rows_frame(&bounds, part_with_delta, last_curr_key); + assert_cache_key_eq(first_frame_start, 1); + assert!(last_frame_end.is_largest()); + } + } + } + + mod range_frame_tests { + use risingwave_common::types::{data_types, DataType, Interval}; + use risingwave_common::util::sort_util::OrderType; + use risingwave_expr::window_function::RangeFrameOffset; + + use super::*; + + fn create_range_frame( + order_data_type: DataType, + order_type: OrderType, + start: FrameBound, + end: FrameBound, + ) -> RangeFrameBounds + where + T: Into, + { + let offset_data_type = match &order_data_type { + t @ data_types::range_frame_numeric!() => t.clone(), + data_types::range_frame_datetime!() => DataType::Interval, + _ => unreachable!(), + }; + + let map_fn = |x: T| { + RangeFrameOffset::new_for_test(x.into(), &order_data_type, &offset_data_type) + }; + let start = start.map(map_fn); + let end = end.map(map_fn); + + RangeFrameBounds { + order_data_type, + order_type, + offset_data_type, + start, + end, + } + } + + #[test] + fn test_calc_logical_for_int64_asc() { + let order_data_type = DataType::Int64; + let order_type = OrderType::ascending(); + + let range_frames = [ + &create_range_frame( + order_data_type.clone(), + order_type, + Preceding(3i64), + Preceding(2i64), + ), + &create_range_frame( + order_data_type.clone(), + order_type, + Preceding(1i64), + Following(2i64), + ), + ]; + + let ord_key_1 = StateKey { + order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::Int64(1)), order_type) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + let ord_key_2 = StateKey { + order_key: memcmp_encoding::encode_value(&Some(ScalarImpl::Int64(3)), order_type) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + + let (logical_first_curr, logical_last_curr) = + calc_logical_curr_for_range_frames(&range_frames, &ord_key_1, &ord_key_2).unwrap(); + assert_eq!( + logical_first_curr.as_normal_expect(), + &Some(ScalarImpl::Int64(-1)) + ); + assert_eq!( + logical_last_curr.as_normal_expect(), + &Some(ScalarImpl::Int64(6)) + ); + + let (first_start, last_end) = + calc_logical_boundary_for_range_frames(&range_frames, &ord_key_1, &ord_key_2) + .unwrap(); + assert_eq!(first_start.as_normal_expect(), &Some(ScalarImpl::Int64(-2))); + assert_eq!(last_end.as_normal_expect(), &Some(ScalarImpl::Int64(5))); + } + + #[test] + fn test_calc_logical_for_timestamp_desc_nulls_first() { + let order_data_type = DataType::Timestamp; + let order_type = OrderType::descending_nulls_first(); + + let range_frames = [&create_range_frame( + order_data_type.clone(), + order_type, + Preceding(Interval::from_month_day_usec(1, 2, 3 * 1000 * 1000)), + Following(Interval::from_month_day_usec(0, 1, 0)), + )]; + + let ord_key_1 = StateKey { + order_key: memcmp_encoding::encode_value( + &Some(ScalarImpl::Timestamp( + "2024-01-28 00:30:00".parse().unwrap(), + )), + order_type, + ) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + let ord_key_2 = StateKey { + order_key: memcmp_encoding::encode_value( + &Some(ScalarImpl::Timestamp( + "2024-01-26 15:47:00".parse().unwrap(), + )), + order_type, + ) + .unwrap(), + pk: OwnedRow::empty().into(), + }; + + let (logical_first_curr, logical_last_curr) = + calc_logical_curr_for_range_frames(&range_frames, &ord_key_1, &ord_key_2).unwrap(); + assert_eq!( + logical_first_curr.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2024-01-29 00:30:00".parse().unwrap() + )) + ); + assert_eq!( + logical_last_curr.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2023-12-24 15:46:57".parse().unwrap() + )) + ); + + let (first_start, last_end) = + calc_logical_boundary_for_range_frames(&range_frames, &ord_key_1, &ord_key_2) + .unwrap(); + assert_eq!( + first_start.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2024-03-01 00:30:03".parse().unwrap() + )) + ); + assert_eq!( + last_end.as_normal_expect(), + &Some(ScalarImpl::Timestamp( + "2024-01-25 15:47:00".parse().unwrap() + )) + ); + } + + fn assert_find_left_right_result_eq( + order_data_type: DataType, + order_type: OrderType, + part_with_delta: DeltaBTreeMap<'_, CacheKey, OwnedRow>, + logical_order_value: ScalarImpl, + expected_left: Sentinelled, + expected_right: Sentinelled, + ) { + let frames = if matches!(order_data_type, DataType::Int32) { + [create_range_frame( + order_data_type.clone(), + order_type, + Preceding(0), // this doesn't matter here + Following(0), // this doesn't matter here + )] + } else { + panic!() + }; + let range_frames = frames.iter().collect::>(); + let logical_order_value = Some(logical_order_value); + let cache_key_pk_len = 1; + + let find_left_res = find_left_for_range_frames( + &range_frames, + part_with_delta, + &logical_order_value, + cache_key_pk_len, + ) + .clone(); + assert_eq!( + find_left_res.map(|x| x.pk.0.into_iter().next().unwrap().unwrap()), + expected_left + ); + + let find_right_res = find_right_for_range_frames( + &range_frames, + part_with_delta, + &logical_order_value, + cache_key_pk_len, + ) + .clone(); + assert_eq!( + find_right_res.map(|x| x.pk.0.into_iter().next().unwrap().unwrap()), + expected_right + ); + } + + #[test] + fn test_insert_delta_only() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(); + let delta = create_delta!( + order_type, + [ + (1, 1, Insert), + (1, 11, Insert), + (3, 3, Insert), + (5, 5, Insert) + ] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(2), + ScalarImpl::from(3).into(), + ScalarImpl::from(11).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(5), + ScalarImpl::from(5).into(), + ScalarImpl::from(5).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(6), + ScalarImpl::from(5).into(), + ScalarImpl::from(5).into(), + ); + } + + #[test] + fn test_simple() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [(2, 2), (3, 3), (4, 4), (5, 5), (6, 6)]); + let delta = create_delta!( + order_type, + [(2, 2, Insert), (2, 22, Insert), (3, 3, Delete)] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(0), + ScalarImpl::from(2).into(), + ScalarImpl::from(2).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(2), + ScalarImpl::from(2).into(), + ScalarImpl::from(22).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(3), + ScalarImpl::from(4).into(), + ScalarImpl::from(22).into(), + ); + } + + #[test] + fn test_empty_with_sentinels() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [..., , ...]); + let delta = create_delta!(order_type, [(1, 1, Insert), (2, 2, Insert)]); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(0), + Sentinelled::Smallest, + Sentinelled::Smallest, + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(1), + Sentinelled::Smallest, + ScalarImpl::from(1).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(2), + ScalarImpl::from(2).into(), + Sentinelled::Largest, + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(3), + Sentinelled::Largest, + Sentinelled::Largest, + ); + } + + #[test] + fn test_with_left_sentinels() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [..., (2, 2), (4, 4), (5, 5)]); + let delta = create_delta!( + order_type, + [ + (1, 1, Insert), + (2, 2, Insert), + (4, 44, Insert), + (5, 5, Delete) + ] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(1), + Sentinelled::Smallest, + ScalarImpl::from(1).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(4), + ScalarImpl::from(4).into(), + ScalarImpl::from(44).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(5), + ScalarImpl::from(44).into(), + ScalarImpl::from(44).into(), + ); + } + + #[test] + fn test_with_right_sentinel() { + let order_data_type = DataType::Int32; + let order_type = OrderType::ascending(); + + let cache = create_cache!(order_type, [(2, 2), (4, 4), (5, 5), ...]); + let delta = create_delta!( + order_type, + [ + (1, 1, Insert), + (2, 2, Insert), + (4, 44, Insert), + (5, 5, Delete) + ] + ); + let part_with_delta = DeltaBTreeMap::new(&cache, &delta); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(1), + ScalarImpl::from(1).into(), + ScalarImpl::from(1).into(), + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(4), + ScalarImpl::from(4).into(), + Sentinelled::Largest, + ); + + assert_find_left_right_result_eq( + order_data_type.clone(), + order_type, + part_with_delta, + ScalarImpl::from(5), + Sentinelled::Largest, + Sentinelled::Largest, + ); + } + } +} diff --git a/src/stream/src/executor/over_window/general.rs b/src/stream/src/executor/over_window/general.rs index 0ba4808b93624..a7245c57f368c 100644 --- a/src/stream/src/executor/over_window/general.rs +++ b/src/stream/src/executor/over_window/general.rs @@ -23,6 +23,7 @@ use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::stream_record::Record; use risingwave_common::array::{Op, RowRef, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::row::{OwnedRow, Row, RowExt}; use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy; use risingwave_common::types::{DataType, DefaultOrdered}; @@ -32,6 +33,7 @@ use risingwave_common::util::sort_util::OrderType; use risingwave_expr::window_function::{ create_window_state, StateKey, WindowFuncCall, WindowStates, }; +use risingwave_storage::row_serde::row_serde_util::serialize_pk_with_vnode; use risingwave_storage::StateStore; use super::over_partition::{ @@ -40,12 +42,13 @@ use super::over_partition::{ }; use crate::cache::{new_unbounded, ManagedLruCache}; use crate::common::metrics::MetricsInfo; +use crate::common::table::state_table::StateTable; use crate::common::StreamChunkBuilder; use crate::executor::monitor::StreamingMetrics; -use crate::executor::test_utils::prelude::StateTable; +use crate::executor::over_window::over_partition::AffectedRange; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, - StreamExecutorError, StreamExecutorResult, + expect_first_barrier, ActorContextRef, Execute, Executor, Message, StreamExecutorError, + StreamExecutorResult, }; use crate::task::AtomicU64Ref; @@ -55,14 +58,14 @@ use crate::task::AtomicU64Ref; /// - State table schema = output schema, state table pk = `partition key | order key | input pk`. /// - Output schema = input schema + window function results. pub struct OverWindowExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } struct ExecutorInner { actor_ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, calls: Vec, partition_key_indices: Vec, order_key_indices: Vec, @@ -96,22 +99,10 @@ struct ExecutionStats { cache_lookup: u64, } -impl Executor for OverWindowExecutor { +impl Execute for OverWindowExecutor { fn execute(self: Box) -> crate::executor::BoxedMessageStream { self.executor_inner().boxed() } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> crate::executor::PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } } impl ExecutorInner { @@ -143,10 +134,10 @@ impl ExecutorInner { pub struct OverWindowExecutorArgs { pub actor_ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, + pub schema: Schema, pub calls: Vec, pub partition_key_indices: Vec, pub order_key_indices: Vec, @@ -162,7 +153,7 @@ pub struct OverWindowExecutorArgs { impl OverWindowExecutor { pub fn new(args: OverWindowExecutorArgs) -> Self { - let input_info = args.input.info(); + let input_info = args.input.info().clone(); let input_schema = &input_info.schema; let has_unbounded_frame = args @@ -180,7 +171,7 @@ impl OverWindowExecutor { let order_key_data_types = args .order_key_indices .iter() - .map(|i| input_schema.fields()[*i].data_type.clone()) + .map(|i| input_schema[*i].data_type()) .collect(); let state_key_to_table_sub_pk_proj = RowConverter::calc_state_key_to_table_sub_pk_proj( @@ -193,7 +184,7 @@ impl OverWindowExecutor { input: args.input, inner: ExecutorInner { actor_ctx: args.actor_ctx, - info: args.info, + schema: args.schema, calls: args.calls, partition_key_indices: args.partition_key_indices, order_key_indices: args.order_key_indices, @@ -324,9 +315,9 @@ impl OverWindowExecutor { } // `input pk` => `Record` - let mut key_change_update_buffer = BTreeMap::new(); - let mut chunk_builder = - StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types()); + let mut key_change_update_buffer: BTreeMap, Record> = + BTreeMap::new(); + let mut chunk_builder = StreamChunkBuilder::new(this.chunk_size, this.schema.data_types()); // Prepare things needed by metrics. let actor_id = this.actor_ctx.id.to_string(); @@ -382,7 +373,15 @@ impl OverWindowExecutor { yield chunk; } } - _ => panic!("other cases should not exist"), + (existed, record) => { + let vnode = this.state_table.compute_vnode_by_pk(&key.pk); + let raw_key = serialize_pk_with_vnode( + &key.pk, + this.state_table.pk_serde(), + vnode, + ); + panic!("other cases should not exist. raw_key: {:?}, existed: {:?}, new: {:?}", raw_key, existed, record); + } } } else { key_change_update_buffer.insert(pk, record); @@ -481,7 +480,13 @@ impl OverWindowExecutor { let mut accessed_range: Option> = None; - for (first_frame_start, first_curr_key, last_curr_key, last_frame_end) in affected_ranges { + for AffectedRange { + first_frame_start, + first_curr_key, + last_curr_key, + last_frame_end, + } in affected_ranges + { assert!(first_frame_start <= first_curr_key); assert!(first_curr_key <= last_curr_key); assert!(last_curr_key <= last_frame_end); diff --git a/src/stream/src/executor/over_window/mod.rs b/src/stream/src/executor/over_window/mod.rs index ea43672a7f564..eeeb70de5ac3f 100644 --- a/src/stream/src/executor/over_window/mod.rs +++ b/src/stream/src/executor/over_window/mod.rs @@ -13,6 +13,7 @@ // limitations under the License. mod eowc; +mod frame_finder; mod general; mod over_partition; diff --git a/src/stream/src/executor/over_window/over_partition.rs b/src/stream/src/executor/over_window/over_partition.rs index 7a395821f6030..a759e9e7334e3 100644 --- a/src/stream/src/executor/over_window/over_partition.rs +++ b/src/stream/src/executor/over_window/over_partition.rs @@ -20,18 +20,22 @@ use std::marker::PhantomData; use std::ops::{Bound, RangeInclusive}; use delta_btree_map::{Change, DeltaBTreeMap}; +use educe::Educe; use futures_async_stream::for_await; use risingwave_common::array::stream_record::Record; use risingwave_common::estimate_size::collections::EstimatedBTreeMap; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::session_config::OverWindowCachePolicy as CachePolicy; -use risingwave_common::types::Sentinelled; -use risingwave_expr::window_function::{FrameBounds, StateKey, WindowFuncCall}; +use risingwave_common::types::{Datum, Sentinelled}; +use risingwave_expr::window_function::{ + RangeFrameBounds, RowsFrameBounds, StateKey, WindowFuncCall, +}; use risingwave_storage::store::PrefetchOptions; use risingwave_storage::StateStore; use super::general::RowConverter; -use crate::executor::test_utils::prelude::StateTable; +use crate::common::table::state_table::StateTable; +use crate::executor::over_window::frame_finder::*; use crate::executor::StreamExecutorResult; pub(super) type CacheKey = Sentinelled; @@ -221,6 +225,40 @@ pub(super) struct OverPartitionStats { pub right_miss_count: u64, } +/// [`AffectedRange`] represents a range of keys that are affected by a delta. +/// The [`CacheKey`] fields are keys in the partition range cache + delta, which is +/// represented by [`DeltaBTreeMap`]. +/// +/// - `first_curr_key` and `last_curr_key` are the current keys of the first and the last +/// windows affected. They are used to pinpoint the bounds where state needs to be updated. +/// - `first_frame_start` and `last_frame_end` are the frame start and end of the first and +/// the last windows affected. They are used to pinpoint the bounds where state needs to be +/// included for computing the new state. +#[derive(Debug, Educe)] +#[educe(Clone, Copy)] +pub(super) struct AffectedRange<'cache> { + pub first_frame_start: &'cache CacheKey, + pub first_curr_key: &'cache CacheKey, + pub last_curr_key: &'cache CacheKey, + pub last_frame_end: &'cache CacheKey, +} + +impl<'cache> AffectedRange<'cache> { + fn new( + first_frame_start: &'cache CacheKey, + first_curr_key: &'cache CacheKey, + last_curr_key: &'cache CacheKey, + last_frame_end: &'cache CacheKey, + ) -> Self { + Self { + first_frame_start, + first_curr_key, + last_curr_key, + last_frame_end, + } + } +} + /// A wrapper of [`PartitionCache`] that provides helper methods to manipulate the cache. /// By putting this type inside `private` module, we can avoid misuse of the internal fields and /// methods. @@ -229,7 +267,12 @@ pub(super) struct OverPartition<'a, S: StateStore> { range_cache: &'a mut PartitionCache, cache_policy: CachePolicy, - calls: &'a [WindowFuncCall], + /// The `ROWS` frame that is the union of all `ROWS` frames of all window functions in this + /// over window executor. + super_rows_frame_bounds: RowsFrameBounds, + range_frames: Vec<&'a RangeFrameBounds>, + start_is_unbounded: bool, + end_is_unbounded: bool, row_conv: RowConverter<'a>, stats: OverPartitionStats, @@ -248,12 +291,33 @@ impl<'a, S: StateStore> OverPartition<'a, S> { calls: &'a [WindowFuncCall], row_conv: RowConverter<'a>, ) -> Self { + let rows_frames = calls + .iter() + .filter_map(|call| call.frame.bounds.as_rows()) + .collect::>(); + // TODO(rc): maybe should avoid repeated merging + let super_rows_frame_bounds = merge_rows_frames(&rows_frames); + let range_frames = calls + .iter() + .filter_map(|call| call.frame.bounds.as_range()) + .collect::>(); + + let start_is_unbounded = calls + .iter() + .any(|call| call.frame.bounds.start_is_unbounded()); + let end_is_unbounded = calls + .iter() + .any(|call| call.frame.bounds.end_is_unbounded()); + Self { this_partition_key, range_cache: cache, cache_policy, - calls, + super_rows_frame_bounds, + range_frames, + start_is_unbounded, + end_is_unbounded, row_conv, stats: Default::default(), @@ -352,19 +416,14 @@ impl<'a, S: StateStore> OverPartition<'a, S> { /// Find all ranges in the partition that are affected by the given delta. /// The returned ranges are guaranteed to be sorted and non-overlapping. All keys in the ranges - /// are guaranteed to be cached. + /// are guaranteed to be cached, which means they should be [`Sentinelled::Normal`]s. pub async fn find_affected_ranges<'s, 'cache>( &'s mut self, table: &'_ StateTable, delta: &'cache PartitionDelta, ) -> StreamExecutorResult<( DeltaBTreeMap<'cache, CacheKey, OwnedRow>, - Vec<( - &'cache CacheKey, - &'cache CacheKey, - &'cache CacheKey, - &'cache CacheKey, - )>, + Vec>, )> where 's: 'cache, @@ -372,53 +431,49 @@ impl<'a, S: StateStore> OverPartition<'a, S> { let delta_first = delta.first_key_value().unwrap().0.as_normal_expect(); let delta_last = delta.last_key_value().unwrap().0.as_normal_expect(); + let range_frame_logical_curr = + calc_logical_curr_for_range_frames(&self.range_frames, delta_first, delta_last); + if self.cache_policy.is_full() { // ensure everything is in the cache self.extend_cache_to_boundary(table).await?; } else { + // TODO(rc): later we should extend cache using `self.super_rows_frame_bounds` and + // `range_frame_logical_curr` as hints. + // ensure the cache covers all delta (if possible) self.extend_cache_by_range(table, delta_first..=delta_last) .await?; } loop { - // Terminateability: `extend_cache_leftward_by_n` and `extend_cache_rightward_by_n` keep + // TERMINATEABILITY: `extend_cache_leftward_by_n` and `extend_cache_rightward_by_n` keep // pushing the cache to the boundary of current partition. In these two methods, when // any side of boundary is reached, the sentinel key will be removed, so finally - // `self::find_affected_ranges` will return ranges without any sentinels. - - let (left_reached_sentinel, right_reached_sentinel) = { - // SAFETY: Here we shortly borrow the range cache and turn the reference into a - // `'cache` one to bypass the borrow checker. This is safe because we only return - // the reference once we don't need to do any further mutation. - let cache_inner = unsafe { &*(self.range_cache.inner() as *const _) }; - let ranges = - self::find_affected_ranges(self.calls, DeltaBTreeMap::new(cache_inner, delta)); - self.stats.lookup_count += 1; - - if ranges.is_empty() { - // no ranges affected, we're done - return Ok((DeltaBTreeMap::new(cache_inner, delta), ranges)); - } + // `Self::find_affected_ranges_readonly` will return `Ok`. - let left_reached_sentinel = ranges.first().unwrap().0.is_sentinel(); - let right_reached_sentinel = ranges.last().unwrap().3.is_sentinel(); + // SAFETY: Here we shortly borrow the range cache and turn the reference into a + // `'cache` one to bypass the borrow checker. This is safe because we only return + // the reference once we don't need to do any further mutation. + let cache_inner = unsafe { &*(self.range_cache.inner() as *const _) }; + let part_with_delta = DeltaBTreeMap::new(cache_inner, delta); - if !left_reached_sentinel && !right_reached_sentinel { - // all affected ranges are already cached, we're done - return Ok((DeltaBTreeMap::new(cache_inner, delta), ranges)); - } + self.stats.lookup_count += 1; + let res = self + .find_affected_ranges_readonly(part_with_delta, range_frame_logical_curr.as_ref()); - (left_reached_sentinel, right_reached_sentinel) + let (need_extend_leftward, need_extend_rightward) = match res { + Ok(ranges) => return Ok((part_with_delta, ranges)), + Err(cache_extend_hint) => cache_extend_hint, }; - if left_reached_sentinel { + if need_extend_leftward { self.stats.left_miss_count += 1; tracing::trace!(partition=?self.this_partition_key, "partition cache left extension triggered"); let left_most = self.cache_real_first_key().unwrap_or(delta_first).clone(); self.extend_cache_leftward_by_n(table, &left_most).await?; } - if right_reached_sentinel { + if need_extend_rightward { self.stats.right_miss_count += 1; tracing::trace!(partition=?self.this_partition_key, "partition cache right extension triggered"); let right_most = self.cache_real_last_key().unwrap_or(delta_last).clone(); @@ -428,6 +483,203 @@ impl<'a, S: StateStore> OverPartition<'a, S> { } } + /// Try to find affected ranges on immutable range cache + delta. If the algorithm reaches + /// any sentinel node in the cache, which means some entries in the affected range may be + /// in the state table, it returns an `Err((bool, bool))` to notify the caller that the + /// left side or the right side or both sides of the cache should be extended. + /// + /// TODO(rc): Currently at most one range will be in the result vector. Ideally we should + /// recognize uncontinuous changes in the delta and find multiple ranges, but that will be + /// too complex for now. + fn find_affected_ranges_readonly<'cache>( + &'_ self, + part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, + range_frame_logical_curr: Option<&(Sentinelled, Sentinelled)>, + ) -> std::result::Result>, (bool, bool)> { + if part_with_delta.first_key().is_none() { + // nothing is left after applying the delta, meaning all entries are deleted + return Ok(vec![]); + } + + let delta_first_key = part_with_delta.delta().first_key_value().unwrap().0; + let delta_last_key = part_with_delta.delta().last_key_value().unwrap().0; + let cache_key_pk_len = delta_first_key.as_normal_expect().pk.len(); + + if part_with_delta.snapshot().is_empty() { + // all existing keys are inserted in the delta + return Ok(vec![AffectedRange::new( + delta_first_key, + delta_first_key, + delta_last_key, + delta_last_key, + )]); + } + + let first_key = part_with_delta.first_key().unwrap(); + let last_key = part_with_delta.last_key().unwrap(); + + let first_curr_key = if self.end_is_unbounded || delta_first_key == first_key { + // If the frame end is unbounded, or, the first key is in delta, then the frame corresponding + // to the first key is always affected. + first_key + } else { + let mut key = find_first_curr_for_rows_frame( + &self.super_rows_frame_bounds, + part_with_delta, + delta_first_key, + ); + + if let Some((logical_first_curr, _)) = range_frame_logical_curr { + let logical_curr = logical_first_curr.as_normal_expect(); // otherwise should go `end_is_unbounded` branch + let new_key = find_left_for_range_frames( + &self.range_frames, + part_with_delta, + logical_curr, + cache_key_pk_len, + ); + key = std::cmp::min(key, new_key); + } + + key + }; + + let last_curr_key = if self.start_is_unbounded || delta_last_key == last_key { + // similar to `first_curr_key` + last_key + } else { + let mut key = find_last_curr_for_rows_frame( + &self.super_rows_frame_bounds, + part_with_delta, + delta_last_key, + ); + + if let Some((_, logical_last_curr)) = range_frame_logical_curr { + let logical_curr = logical_last_curr.as_normal_expect(); // otherwise should go `start_is_unbounded` branch + let new_key = find_right_for_range_frames( + &self.range_frames, + part_with_delta, + logical_curr, + cache_key_pk_len, + ); + key = std::cmp::max(key, new_key); + } + + key + }; + + { + // We quickly return if there's any sentinel in `[first_curr_key, last_curr_key]`, + // just for the sake of simplicity. + let mut need_extend_leftward = false; + let mut need_extend_rightward = false; + for key in [first_curr_key, last_curr_key] { + if key.is_smallest() { + need_extend_leftward = true; + } else if key.is_largest() { + need_extend_rightward = true; + } + } + if need_extend_leftward || need_extend_rightward { + return Err((need_extend_leftward, need_extend_rightward)); + } + } + + // From now on we definitely have two normal `curr_key`s. + + if first_curr_key > last_curr_key { + // Note that we cannot move the this check before the above block, because for example, + // if the range cache contains `[Smallest, 5, Largest]`, and the delta contains only + // `Delete 5`, the frame is `RANGE BETWEEN CURRENT ROW AND CURRENT ROW`, then + // `first_curr_key` will be `Largest`, `last_curr_key` will be `Smallest`, in this case + // there may be some other entries with order value `5` in the table, which should be + // *affected*. + return Ok(vec![]); + } + + let range_frame_logical_boundary = calc_logical_boundary_for_range_frames( + &self.range_frames, + first_curr_key.as_normal_expect(), + last_curr_key.as_normal_expect(), + ); + + let first_frame_start = if self.start_is_unbounded || first_curr_key == first_key { + // If the frame start is unbounded, or, the first curr key is the first key, then the first key + // always need to be included in the affected range. + first_key + } else { + let mut key = find_frame_start_for_rows_frame( + &self.super_rows_frame_bounds, + part_with_delta, + first_curr_key, + ); + + if let Some((logical_first_start, _)) = range_frame_logical_boundary.as_ref() { + let logical_boundary = logical_first_start.as_normal_expect(); // otherwise should go `end_is_unbounded` branch + let new_key = find_left_for_range_frames( + &self.range_frames, + part_with_delta, + logical_boundary, + cache_key_pk_len, + ); + key = std::cmp::min(key, new_key); + } + + key + }; + assert!(first_frame_start <= first_curr_key); + + let last_frame_end = if self.end_is_unbounded || last_curr_key == last_key { + // similar to `first_frame_start` + last_key + } else { + let mut key = find_frame_end_for_rows_frame( + &self.super_rows_frame_bounds, + part_with_delta, + last_curr_key, + ); + + if let Some((_, logical_last_end)) = range_frame_logical_boundary.as_ref() { + let logical_boundary = logical_last_end.as_normal_expect(); // otherwise should go `end_is_unbounded` branch + let new_key = find_right_for_range_frames( + &self.range_frames, + part_with_delta, + logical_boundary, + cache_key_pk_len, + ); + key = std::cmp::max(key, new_key); + } + + key + }; + assert!(last_frame_end >= last_curr_key); + + let mut need_extend_leftward = false; + let mut need_extend_rightward = false; + for key in [ + first_curr_key, + last_curr_key, + first_frame_start, + last_frame_end, + ] { + if key.is_smallest() { + need_extend_leftward = true; + } else if key.is_largest() { + need_extend_rightward = true; + } + } + + if need_extend_leftward || need_extend_rightward { + Err((need_extend_leftward, need_extend_rightward)) + } else { + Ok(vec![AffectedRange::new( + first_frame_start, + first_curr_key, + last_curr_key, + last_frame_end, + )]) + } + } + async fn extend_cache_to_boundary( &mut self, table: &StateTable, @@ -760,577 +1012,3 @@ impl<'a, S: StateStore> OverPartition<'a, S> { Ok(()) } } - -/// Find all affected ranges in the given partition with delta. -/// -/// # Returns -/// -/// `Vec<(first_frame_start, first_curr_key, last_curr_key, last_frame_end_incl)>` -/// -/// Each affected range is a union of many small window frames affected by some adajcent -/// keys in the delta. -/// -/// Example: -/// - frame 1: `rows between 2 preceding and current row` -/// - frame 2: `rows between 1 preceding and 2 following` -/// - partition: `[1, 2, 4, 5, 7, 8, 9, 10, 11, 12, 14]` -/// - delta: `[3, 4, 15]` -/// - affected ranges: `[(1, 1, 7, 9), (10, 12, 15, 15)]` -/// -/// TODO(rc): -/// Note that, since we assume input chunks have data locality on order key columns, we now only -/// calculate one single affected range. So the affected ranges in the above example will be -/// `(1, 1, 15, 15)`. Later we may optimize this. -fn find_affected_ranges<'cache>( - calls: &'_ [WindowFuncCall], - part_with_delta: DeltaBTreeMap<'cache, CacheKey, OwnedRow>, -) -> Vec<( - &'cache CacheKey, - &'cache CacheKey, - &'cache CacheKey, - &'cache CacheKey, -)> { - // XXX(rc): NOTE FOR DEVS - // Must carefully consider the sentinel keys in the cache when extending this function to - // support `RANGE` and `GROUPS` frames later. May introduce a return value variant to clearly - // tell the caller that there exists at least one affected range that touches the sentinel. - - if part_with_delta.first_key().is_none() { - // all keys are deleted in the delta - return vec![]; - } - - let delta_first_key = part_with_delta.delta().first_key_value().unwrap().0; - let delta_last_key = part_with_delta.delta().last_key_value().unwrap().0; - - if part_with_delta.snapshot().is_empty() { - // all existing keys are inserted in the delta - return vec![( - delta_first_key, - delta_first_key, - delta_last_key, - delta_last_key, - )]; - } - - let first_key = part_with_delta.first_key().unwrap(); - let last_key = part_with_delta.last_key().unwrap(); - - let start_is_unbounded = calls - .iter() - .any(|call| call.frame.bounds.start_is_unbounded()); - let end_is_unbounded = calls - .iter() - .any(|call| call.frame.bounds.end_is_unbounded()); - - // NOTE: Don't be too clever! Here we must calculate `first_frame_start` after calculating - // `first_curr_key`, because the correct calculation of `first_frame_start` depends on - // `first_curr_key` which is the MINIMUM of all `first_curr_key`s of all frames of all window - // function calls. - - let first_curr_key = if end_is_unbounded || delta_first_key == first_key { - // If the frame end is unbounded, or, the first key is in delta, then the frame corresponding - // to the first key is always affected. - first_key - } else { - let mut min_first_curr_key = &Sentinelled::Largest; - - for call in calls { - let key = match &call.frame.bounds { - FrameBounds::Rows(bounds) => { - let mut cursor = part_with_delta.lower_bound(Bound::Included(delta_first_key)); - for _ in 0..bounds.end.n_following_rows().unwrap() { - // Note that we have to move before check, to handle situation where the - // cursor is at ghost position at first. - cursor.move_prev(); - if cursor.position().is_ghost() { - break; - } - } - cursor.key().unwrap_or(first_key) - } - }; - min_first_curr_key = min_first_curr_key.min(key); - if min_first_curr_key == first_key { - // if we already pushed the affected curr key to the first key, no more pushing is needed - break; - } - } - - min_first_curr_key - }; - - let first_frame_start = if start_is_unbounded || first_curr_key == first_key { - // If the frame start is unbounded, or, the first curr key is the first key, then the first key - // always need to be included in the affected range. - first_key - } else { - let mut min_frame_start = &Sentinelled::Largest; - - for call in calls { - let key = match &call.frame.bounds { - FrameBounds::Rows(bounds) => { - let mut cursor = part_with_delta.find(first_curr_key).unwrap(); - for _ in 0..bounds.start.n_preceding_rows().unwrap() { - cursor.move_prev(); - if cursor.position().is_ghost() { - break; - } - } - cursor.key().unwrap_or(first_key) - } - }; - min_frame_start = min_frame_start.min(key); - if min_frame_start == first_key { - // if we already pushed the affected frame start to the first key, no more pushing is needed - break; - } - } - - min_frame_start - }; - - let last_curr_key = if start_is_unbounded || delta_last_key == last_key { - last_key - } else { - let mut max_last_curr_key = &Sentinelled::Smallest; - - for call in calls { - let key = match &call.frame.bounds { - FrameBounds::Rows(bounds) => { - let mut cursor = part_with_delta.upper_bound(Bound::Included(delta_last_key)); - for _ in 0..bounds.start.n_preceding_rows().unwrap() { - cursor.move_next(); - if cursor.position().is_ghost() { - break; - } - } - cursor.key().unwrap_or(last_key) - } - }; - max_last_curr_key = max_last_curr_key.max(key); - if max_last_curr_key == last_key { - // if we already pushed the affected curr key to the last key, no more pushing is needed - break; - } - } - - max_last_curr_key - }; - - let last_frame_end = if end_is_unbounded || last_curr_key == last_key { - last_key - } else { - let mut max_frame_end = &Sentinelled::Smallest; - - for call in calls { - let key = match &call.frame.bounds { - FrameBounds::Rows(bounds) => { - let mut cursor = part_with_delta.find(last_curr_key).unwrap(); - for _ in 0..bounds.end.n_following_rows().unwrap() { - cursor.move_next(); - if cursor.position().is_ghost() { - break; - } - } - cursor.key().unwrap_or(last_key) - } - }; - max_frame_end = max_frame_end.max(key); - if max_frame_end == last_key { - // if we already pushed the affected frame end to the last key, no more pushing is needed - break; - } - } - - max_frame_end - }; - - if first_curr_key > last_curr_key { - // all affected keys are deleted in the delta - return vec![]; - } - - vec![( - first_frame_start, - first_curr_key, - last_curr_key, - last_frame_end, - )] -} - -#[cfg(test)] -mod find_affected_ranges_tests { - //! Function `find_affected_ranges` is important enough to deserve its own test module. We must - //! test it thoroughly. - - use itertools::Itertools; - use risingwave_common::types::{DataType, ScalarImpl}; - use risingwave_expr::aggregate::{AggArgs, AggKind}; - use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncKind}; - - use super::*; - - fn create_call(frame: Frame) -> WindowFuncCall { - WindowFuncCall { - kind: WindowFuncKind::Aggregate(AggKind::Sum), - args: AggArgs::Unary(DataType::Int32, 0), - return_type: DataType::Int32, - frame, - } - } - - macro_rules! create_cache { - (..., $( $pk:literal ),* , ...) => { - { - let mut cache = create_cache!( $( $pk ),* ); - cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); - cache.insert(CacheKey::Largest, OwnedRow::empty().into()); - cache - } - }; - (..., $( $pk:literal ),*) => { - { - let mut cache = create_cache!( $( $pk ),* ); - cache.insert(CacheKey::Smallest, OwnedRow::empty().into()); - cache - } - }; - ($( $pk:literal ),* , ...) => { - { - let mut cache = create_cache!( $( $pk ),* ); - cache.insert(CacheKey::Largest, OwnedRow::empty().into()); - cache - } - }; - ($( $pk:literal ),*) => { - { - #[allow(unused_mut)] - let mut cache = BTreeMap::new(); - $( - cache.insert( - CacheKey::Normal( - StateKey { - // order key doesn't matter here - order_key: vec![].into(), - pk: OwnedRow::new(vec![Some($pk.into())]).into(), - }, - ), - // value row doesn't matter here - OwnedRow::empty(), - ); - )* - cache - } - }; - } - - macro_rules! create_change { - (Delete) => { - Change::Delete - }; - (Insert) => { - Change::Insert(OwnedRow::empty()) - }; - } - - macro_rules! create_delta { - ($(( $pk:literal, $change:ident )),* $(,)?) => { - { - #[allow(unused_mut)] - let mut delta = BTreeMap::new(); - $( - delta.insert( - CacheKey::Normal( - StateKey { - // order key doesn't matter here - order_key: vec![].into(), - pk: OwnedRow::new(vec![Some($pk.into())]).into(), - }, - ), - // value row doesn't matter here - create_change!( $change ), - ); - )* - delta - } - }; - } - - fn assert_ranges_eq( - result: Vec<(&CacheKey, &CacheKey, &CacheKey, &CacheKey)>, - expected: impl IntoIterator, - ) { - result - .into_iter() - .zip_eq(expected) - .for_each(|(result, expected)| { - assert_eq!( - result.0.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(expected.0)]) - ); - assert_eq!( - result.1.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(expected.1)]) - ); - assert_eq!( - result.2.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(expected.2)]) - ); - assert_eq!( - result.3.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(expected.3)]) - ); - }) - } - - #[test] - fn test_insert_delta_only() { - let cache = create_cache!(); - let delta = create_delta!((1, Insert), (2, Insert), (3, Insert)); - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(2), - FrameBound::Preceding(1), - ))]; - let affected_ranges = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)); - assert_ranges_eq(affected_ranges, [(1.into(), 1.into(), 3.into(), 3.into())]); - } - - #[test] - fn test_simple() { - let cache = create_cache!(1, 2, 3, 4, 5, 6); - let delta = create_delta!((2, Insert), (3, Delete)); - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(2), - FrameBound::Preceding(1), - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(1.into(), 2.into(), 5.into(), 5.into())], - ); - } - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(1), - FrameBound::Following(2), - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(1.into(), 1.into(), 4.into(), 6.into())], - ); - } - - { - let calls = vec![create_call(Frame::rows( - FrameBound::CurrentRow, - FrameBound::Following(2), - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(1.into(), 1.into(), 2.into(), 5.into())], - ); - } - } - - #[test] - fn test_multiple_calls() { - let cache = create_cache!(1, 2, 3, 4, 5, 6); - let delta = create_delta!((2, Insert), (3, Delete)); - let calls = vec![ - create_call(Frame::rows( - FrameBound::Preceding(1), - FrameBound::Preceding(1), - )), - create_call(Frame::rows( - FrameBound::Following(1), - FrameBound::Following(1), - )), - ]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(1.into(), 1.into(), 4.into(), 5.into())], - ); - } - - #[test] - fn test_lag_corner_case() { - let cache = create_cache!(1, 2, 3, 4, 5, 6); - let delta = create_delta!((1, Delete), (2, Delete), (3, Delete)); - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(1), - FrameBound::Preceding(1), - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(4.into(), 4.into(), 4.into(), 4.into())], - ); - } - - #[test] - fn test_lead_corner_case() { - let cache = create_cache!(1, 2, 3, 4, 5, 6); - let delta = create_delta!((4, Delete), (5, Delete), (6, Delete)); - let calls = vec![create_call(Frame::rows( - FrameBound::Following(1), - FrameBound::Following(1), - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(3.into(), 3.into(), 3.into(), 3.into())], - ); - } - - #[test] - fn test_lag_lead_offset_0_corner_case_1() { - let cache = create_cache!(1, 2, 3, 4); - let delta = create_delta!((2, Delete), (3, Delete)); - let calls = vec![create_call(Frame::rows( - FrameBound::CurrentRow, - FrameBound::CurrentRow, - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [], - ); - } - - #[test] - fn test_lag_lead_offset_0_corner_case_2() { - let cache = create_cache!(1, 2, 3, 4, 5); - let delta = create_delta!((2, Delete), (3, Insert), (4, Delete)); - let calls = vec![create_call(Frame::rows( - FrameBound::CurrentRow, - FrameBound::CurrentRow, - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(3.into(), 3.into(), 3.into(), 3.into())], - ); - } - - #[test] - fn test_empty_with_sentinels() { - let cache: BTreeMap, OwnedRow> = create_cache!(..., , ...); - let delta = create_delta!((1, Insert), (2, Insert)); - - { - let calls = vec![create_call(Frame::rows( - FrameBound::CurrentRow, - FrameBound::CurrentRow, - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(1.into(), 1.into(), 2.into(), 2.into())], - ); - } - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(1), - FrameBound::Preceding(1), - ))]; - let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0]; - assert!(range.0.is_smallest()); - assert_eq!( - range.1.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(1.into())]) - ); - assert!(range.2.is_largest()); - assert!(range.3.is_largest()); - } - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Following(1), - FrameBound::Following(3), - ))]; - let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0]; - assert!(range.0.is_smallest()); - assert!(range.1.is_smallest()); - assert_eq!( - range.2.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(2.into())]) - ); - assert!(range.3.is_largest()); - } - } - - #[test] - fn test_with_left_sentinel() { - let cache = create_cache!(..., 2, 4, 5, 8); - let delta = create_delta!((3, Insert), (4, Insert), (8, Delete)); - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Following(1), - FrameBound::Following(1), - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(2.into(), 2.into(), 5.into(), 5.into())], - ); - } - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(1), - FrameBound::Following(1), - ))]; - let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0]; - assert!(range.0.is_smallest()); - assert_eq!( - range.1.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(2.into())]) - ); - assert_eq!( - range.2.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(5.into())]) - ); - assert_eq!( - range.3.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(5.into())]) - ); - } - } - - #[test] - fn test_with_right_sentinel() { - let cache = create_cache!(1, 2, 4, 5, 8, ...); - let delta = create_delta!((3, Insert), (4, Insert), (5, Delete)); - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(1), - FrameBound::Preceding(1), - ))]; - assert_ranges_eq( - find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta)), - [(2.into(), 3.into(), 8.into(), 8.into())], - ); - } - - { - let calls = vec![create_call(Frame::rows( - FrameBound::Preceding(1), - FrameBound::Following(1), - ))]; - let range = find_affected_ranges(&calls, DeltaBTreeMap::new(&cache, &delta))[0]; - assert_eq!( - range.0.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(1.into())]) - ); - assert_eq!( - range.1.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(2.into())]) - ); - assert_eq!( - range.2.as_normal_expect().pk.0, - OwnedRow::new(vec![Some(8.into())]) - ); - assert!(range.3.is_largest()); - } - } -} diff --git a/src/stream/src/executor/project.rs b/src/stream/src/executor/project.rs index 00be2857b3a2e..6e70539bf1dd9 100644 --- a/src/stream/src/executor/project.rs +++ b/src/stream/src/executor/project.rs @@ -14,32 +14,30 @@ use std::fmt::{Debug, Formatter}; -use auto_enums::auto_enum; -use futures::stream::Stream; +use futures::StreamExt; +use futures_async_stream::try_stream; use multimap::MultiMap; use risingwave_common::array::StreamChunk; -use risingwave_common::catalog::Schema; use risingwave_common::row::{Row, RowExt}; -use risingwave_common::types::ToOwnedDatum; +use risingwave_common::types::{ScalarImpl, ToOwnedDatum}; use risingwave_common::util::iter_util::ZipEqFast; -use risingwave_common::util::{RwFutureExt, RwTryStreamExt}; use risingwave_expr::expr::NonStrictExpression; -use super::*; +use super::{ + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, StreamExecutorError, + StreamExecutorResult, Watermark, +}; /// `ProjectExecutor` project data with the `expr`. The `expr` takes a chunk of data, /// and returns a new data chunk. And then, `ProjectExecutor` will insert, delete /// or update element into next operator according to the result of the expression. pub struct ProjectExecutor { - input: BoxedExecutor, + input: Executor, inner: Inner, - /// The mutable parts of inner fields. - vars: ExecutionVars, } struct Inner { _ctx: ActorContextRef, - info: ExecutorInfo, /// Expressions of the current projection. exprs: Vec, @@ -48,23 +46,19 @@ struct Inner { watermark_derivations: MultiMap, /// Indices of nondecreasing expressions in the expression list. nondecreasing_expr_indices: Vec, + /// Last seen values of nondecreasing expressions, buffered to periodically produce watermarks. + last_nondec_expr_values: Vec>, /// the selectivity threshold which should be in `[0,1]`. for the chunk with selectivity less /// than the threshold, the Project executor will construct a new chunk before expr evaluation, materialize_selectivity_threshold: f64, } -struct ExecutionVars { - /// Last seen values of nondecreasing expressions, buffered to periodically produce watermarks. - last_nondec_expr_values: Vec>, -} - impl ProjectExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, + input: Executor, exprs: Vec, watermark_derivations: MultiMap, nondecreasing_expr_indices: Vec, @@ -75,14 +69,11 @@ impl ProjectExecutor { input, inner: Inner { _ctx: ctx, - info, exprs, watermark_derivations, nondecreasing_expr_indices, - materialize_selectivity_threshold, - }, - vars: ExecutionVars { last_nondec_expr_values: vec![None; n_nondecreasing_exprs], + materialize_selectivity_threshold, }, } } @@ -96,21 +87,9 @@ impl Debug for ProjectExecutor { } } -impl Executor for ProjectExecutor { - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } - +impl Execute for ProjectExecutor { fn execute(self: Box) -> BoxedMessageStream { - self.inner.execute(self.input, self.vars).boxed() + self.inner.execute(self.input).boxed() } } @@ -152,118 +131,62 @@ impl Inner { ret.push(derived_watermark); } else { warn!( - "{} derive a NULL watermark with the expression {}!", - self.info.identity, out_col_idx + "a NULL watermark is derived with the expression {}!", + out_col_idx ); } } Ok(ret) } - fn execute( - self, - input: BoxedExecutor, - mut vars: ExecutionVars, - ) -> impl Stream { - let return_types: Vec<_> = self.exprs.iter().map(|expr| expr.return_type()).collect(); - - // Phase 1: only evaluating the expression, which can be concurrent. - - enum Phase1Item { - Chunk(Option), - Barrier(Barrier), - Watermark(Vec), - } - - let this = Arc::new(self); - - let this2 = this.clone(); - - let st = input.execute().map(move |msg| { - let this = this.clone(); + #[try_stream(ok = Message, error = StreamExecutorError)] + async fn execute(mut self, input: Executor) { + #[for_await] + for msg in input.execute() { let msg = msg?; - let is_fence: bool; - #[auto_enum(Future)] - let fut = match msg { - Message::Chunk(chunk) => { - is_fence = false; - async move { - let new_chunk = this.map_filter_chunk(chunk).await?; - Ok(Phase1Item::Chunk(new_chunk)) as StreamExecutorResult<_> + match msg { + Message::Watermark(w) => { + let watermarks = self.handle_watermark(w).await?; + for watermark in watermarks { + yield Message::Watermark(watermark) } } - Message::Watermark(watermark) => { - is_fence = false; - async move { - let watermarks = this.handle_watermark(watermark).await?; - Ok(Phase1Item::Watermark(watermarks)) - } - } - Message::Barrier(barrier) => { - is_fence = true; - async { Ok(Phase1Item::Barrier(barrier)) } - } - }; - - let fut = fut.with_fence(is_fence); - - Ok(fut) as StreamExecutorResult<_> - }); - - // Make the phase 1 concurrent. - let st = st.try_buffered_with_fence(16); - - let this = this2; - - // Phase 2: Handle the watermark related logicals, and output them all. The phase is executed one by one. - #[try_stream] - async move { - #[for_await] - for msg in st { - let msg = msg?; - match msg { - Phase1Item::Watermark(watermarks) => { - for watermark in watermarks { - yield Message::Watermark(watermark) - } - } - Phase1Item::Chunk(new_chunk) => match new_chunk { - Some(new_chunk) => { - if !this.nondecreasing_expr_indices.is_empty() { - if let Some((_, first_visible_row)) = new_chunk.rows().next() { - // it's ok to use the first row here, just one chunk delay - first_visible_row - .project(&this.nondecreasing_expr_indices) - .iter() - .enumerate() - .for_each(|(idx, value)| { - vars.last_nondec_expr_values[idx] = - Some(value.to_owned_datum().expect( - "non-decreasing expression should never be NULL", - )); - }); - } + Message::Chunk(chunk) => match self.map_filter_chunk(chunk).await? { + Some(new_chunk) => { + if !self.nondecreasing_expr_indices.is_empty() { + if let Some((_, first_visible_row)) = new_chunk.rows().next() { + // it's ok to use the first row here, just one chunk delay + first_visible_row + .project(&self.nondecreasing_expr_indices) + .iter() + .enumerate() + .for_each(|(idx, value)| { + self.last_nondec_expr_values[idx] = + Some(value.to_owned_datum().expect( + "non-decreasing expression should never be NULL", + )); + }); } - yield Message::Chunk(new_chunk) } - None => continue, - }, - Phase1Item::Barrier(barrier) => { - for (&expr_idx, value) in this - .nondecreasing_expr_indices - .iter() - .zip_eq_fast(&mut vars.last_nondec_expr_values) - { - if let Some(value) = std::mem::take(value) { - yield Message::Watermark(Watermark::new( - expr_idx, - return_types[expr_idx].clone(), - value, - )) - } + yield Message::Chunk(new_chunk) + } + None => continue, + }, + barrier @ Message::Barrier(_) => { + for (&expr_idx, value) in self + .nondecreasing_expr_indices + .iter() + .zip_eq_fast(&mut self.last_nondec_expr_values) + { + if let Some(value) = std::mem::take(value) { + yield Message::Watermark(Watermark::new( + expr_idx, + self.exprs[expr_idx].return_type(), + value, + )) } - yield Message::Barrier(barrier); } + yield barrier; } } } @@ -308,28 +231,20 @@ mod tests { ], }; let pk_indices = vec![0]; - let (mut tx, source) = MockSource::channel(schema, pk_indices); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, pk_indices); let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)"); - let info = ExecutorInfo { - schema: Schema { - fields: vec![Field::unnamed(DataType::Int64)], - }, - pk_indices: vec![], - identity: "ProjectExecutor".to_string(), - }; - - let project = Box::new(ProjectExecutor::new( - ActorContext::create(123), - info, - Box::new(source), + let project = ProjectExecutor::new( + ActorContext::for_test(123), + source, vec![test_expr], MultiMap::new(), vec![], 0.0, - )); - let mut project = project.execute(); + ); + let mut project = project.boxed().execute(); tx.push_barrier(test_epoch(1), false); let barrier = project.next().await.unwrap().unwrap(); @@ -396,34 +311,22 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (mut tx, source) = MockSource::channel(schema, PkIndices::new()); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, PkIndices::new()); let a_expr = build_from_pretty("(add:int8 $0:int8 1:int8)"); let b_expr = build_from_pretty("(subtract:int8 $0:int8 1:int8)"); let c_expr = NonStrictExpression::for_test(DummyNondecreasingExpr); - let info = ExecutorInfo { - schema: Schema { - fields: vec![ - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - ], - }, - pk_indices: vec![], - identity: "ProjectExecutor".to_string(), - }; - - let project = Box::new(ProjectExecutor::new( - ActorContext::create(123), - info, - Box::new(source), + let project = ProjectExecutor::new( + ActorContext::for_test(123), + source, vec![a_expr, b_expr, c_expr], MultiMap::from_iter(vec![(0, 0), (0, 1)].into_iter()), vec![2], 0.0, - )); - let mut project = project.execute(); + ); + let mut project = project.boxed().execute(); tx.push_barrier(test_epoch(1), false); tx.push_int64_watermark(0, 100); diff --git a/src/stream/src/executor/project_set.rs b/src/stream/src/executor/project_set.rs index 5167a71db2cb8..9fadb5949dac2 100644 --- a/src/stream/src/executor/project_set.rs +++ b/src/stream/src/executor/project_set.rs @@ -20,7 +20,6 @@ use futures_async_stream::try_stream; use multimap::MultiMap; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::bail; -use risingwave_common::catalog::Schema; use risingwave_common::row::{Row, RowExt}; use risingwave_common::types::{DataType, Datum, DatumRef, ToOwnedDatum}; use risingwave_common::util::iter_util::ZipEqFast; @@ -28,10 +27,7 @@ use risingwave_expr::expr::{LogReport, NonStrictExpression}; use risingwave_expr::table_function::ProjectSetSelectItem; use super::error::StreamExecutorError; -use super::{ - ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, PkIndicesRef, - StreamExecutorResult, Watermark, -}; +use super::{ActorContextRef, Execute, Executor, Message, StreamExecutorResult, Watermark}; use crate::common::StreamChunkBuilder; const PROJ_ROW_ID_OFFSET: usize = 1; @@ -40,13 +36,12 @@ const PROJ_ROW_ID_OFFSET: usize = 1; /// and returns a new data chunk. And then, `ProjectSetExecutor` will insert, delete /// or update element into next operator according to the result of the expression. pub struct ProjectSetExecutor { - input: BoxedExecutor, + input: Executor, inner: Inner, } struct Inner { _ctx: ActorContextRef, - info: ExecutorInfo, /// Expressions of the current project_section. select_list: Vec, @@ -62,8 +57,7 @@ impl ProjectSetExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, + input: Executor, select_list: Vec, chunk_size: usize, watermark_derivations: MultiMap, @@ -71,7 +65,6 @@ impl ProjectSetExecutor { ) -> Self { let inner = Inner { _ctx: ctx, - info, select_list, chunk_size, watermark_derivations, @@ -90,27 +83,15 @@ impl Debug for ProjectSetExecutor { } } -impl Executor for ProjectSetExecutor { +impl Execute for ProjectSetExecutor { fn execute(self: Box) -> super::BoxedMessageStream { self.inner.execute(self.input).boxed() } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } } impl Inner { #[try_stream(ok = Message, error = StreamExecutorError)] - async fn execute(self, input: BoxedExecutor) { + async fn execute(self, input: Executor) { assert!(!self.select_list.is_empty()); // First column will be `projected_row_id`, which represents the index in the // output table @@ -270,8 +251,8 @@ impl Inner { ret.push(derived_watermark); } else { warn!( - "{} derive a NULL watermark with the expression {}!", - self.info.identity, expr_idx + "a NULL watermark is derived with the expression {}!", + expr_idx ); } } diff --git a/src/stream/src/executor/rearranged_chain.rs b/src/stream/src/executor/rearranged_chain.rs index 1bb83009a203e..0b51bab665d41 100644 --- a/src/stream/src/executor/rearranged_chain.rs +++ b/src/stream/src/executor/rearranged_chain.rs @@ -19,12 +19,9 @@ use futures::stream::select_with_strategy; use futures::{stream, StreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::StreamChunk; -use risingwave_common::catalog::Schema; use super::error::StreamExecutorError; -use super::{ - expect_first_barrier, Barrier, BoxedExecutor, Executor, ExecutorInfo, Message, MessageStream, -}; +use super::{expect_first_barrier, Barrier, Execute, Executor, Message, MessageStream}; use crate::task::{ActorId, CreateMviewProgress}; /// `ChainExecutor` is an executor that enables synchronization between the existing stream and @@ -35,11 +32,9 @@ use crate::task::{ActorId, CreateMviewProgress}; /// [`RearrangedChainExecutor`] resolves the latency problem when creating MV with a huge amount of /// existing data, by rearranging the barrier from the upstream. Check the design doc for details. pub struct RearrangedChainExecutor { - info: ExecutorInfo, + snapshot: Executor, - snapshot: BoxedExecutor, - - upstream: BoxedExecutor, + upstream: Executor, progress: CreateMviewProgress, @@ -84,14 +79,8 @@ impl RearrangedMessage { } impl RearrangedChainExecutor { - pub fn new( - info: ExecutorInfo, - snapshot: BoxedExecutor, - upstream: BoxedExecutor, - progress: CreateMviewProgress, - ) -> Self { + pub fn new(snapshot: Executor, upstream: Executor, progress: CreateMviewProgress) -> Self { Self { - info, snapshot, upstream, actor_id: progress.actor_id(), @@ -288,26 +277,10 @@ impl RearrangedChainExecutor { } } -impl Executor for RearrangedChainExecutor { +impl Execute for RearrangedChainExecutor { fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> ExecutorInfo { - self.info.clone() - } } // TODO: add new unit tests for rearranged chain diff --git a/src/stream/src/executor/receiver.rs b/src/stream/src/executor/receiver.rs index e15acf386faed..436ee8768cb80 100644 --- a/src/stream/src/executor/receiver.rs +++ b/src/stream/src/executor/receiver.rs @@ -17,7 +17,6 @@ use anyhow::Context; use futures::StreamExt; use futures_async_stream::try_stream; use itertools::Itertools; -use risingwave_common::catalog::Schema; use tokio::time::Instant; use super::exchange::input::BoxedInput; @@ -25,17 +24,12 @@ use super::ActorContextRef; use crate::executor::exchange::input::new_input; use crate::executor::monitor::StreamingMetrics; use crate::executor::utils::ActorInputMetrics; -use crate::executor::{ - expect_first_barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, PkIndicesRef, -}; +use crate::executor::{expect_first_barrier, BoxedMessageStream, Execute, Message}; use crate::task::{FragmentId, SharedContext}; /// `ReceiverExecutor` is used along with a channel. After creating a mpsc channel, /// there should be a `ReceiverExecutor` running in the background, so as to push /// messages down to the executors. pub struct ReceiverExecutor { - /// Logical Operator Info - info: ExecutorInfo, - /// Input from upstream. input: BoxedInput, @@ -57,10 +51,7 @@ pub struct ReceiverExecutor { impl std::fmt::Debug for ReceiverExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ReceiverExecutor") - .field("schema", &self.info.schema) - .field("pk_indices", &self.info.pk_indices) - .finish() + f.debug_struct("ReceiverExecutor").finish() } } @@ -68,7 +59,6 @@ impl ReceiverExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, fragment_id: FragmentId, upstream_fragment_id: FragmentId, input: BoxedInput, @@ -78,7 +68,6 @@ impl ReceiverExecutor { ) -> Self { Self { input, - info, actor_context: ctx, upstream_fragment_id, metrics, @@ -94,12 +83,7 @@ impl ReceiverExecutor { use crate::executor::ActorContext; Self::new( - ActorContext::create(114), - ExecutorInfo { - schema: Schema::default(), - pk_indices: vec![], - identity: "ReceiverExecutor".to_string(), - }, + ActorContext::for_test(114), 514, 1919, LocalInput::new(input, 0).boxed_input(), @@ -110,7 +94,7 @@ impl ReceiverExecutor { } } -impl Executor for ReceiverExecutor { +impl Execute for ReceiverExecutor { fn execute(mut self: Box) -> BoxedMessageStream { let actor_id = self.actor_context.id; @@ -207,22 +191,6 @@ impl Executor for ReceiverExecutor { stream.boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> ExecutorInfo { - self.info.clone() - } } #[cfg(test)] @@ -236,13 +204,11 @@ mod tests { use risingwave_pb::stream_plan::update_mutation::MergeUpdate; use super::*; - use crate::executor::{ActorContext, Barrier, Executor, Mutation, UpdateMutation}; + use crate::executor::{ActorContext, Barrier, Execute, Mutation, UpdateMutation}; use crate::task::test_utils::helper_make_local_actor; #[tokio::test] async fn test_configuration_change() { - let schema = Schema { fields: vec![] }; - let actor_id = 233; let (old, new) = (114, 514); // old and new upstream actor id @@ -272,15 +238,8 @@ mod tests { ) .unwrap(); - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "ReceiverExecutor".to_string(), - }; - let receiver = ReceiverExecutor::new( - ActorContext::create(actor_id), - info, + ActorContext::for_test(actor_id), fragment_id, upstream_fragment_id, input, diff --git a/src/stream/src/executor/row_id_gen.rs b/src/stream/src/executor/row_id_gen.rs index e184b626cdd79..12de3e0b88a36 100644 --- a/src/stream/src/executor/row_id_gen.rs +++ b/src/stream/src/executor/row_id_gen.rs @@ -19,23 +19,19 @@ use risingwave_common::array::{ Array, ArrayBuilder, ArrayRef, Op, SerialArrayBuilder, StreamChunk, }; use risingwave_common::buffer::Bitmap; -use risingwave_common::catalog::Schema; use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::types::Serial; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_common::util::row_id::RowIdGenerator; -use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, PkIndicesRef, -}; +use super::{expect_first_barrier, ActorContextRef, Execute, Executor}; use crate::executor::{Message, StreamExecutorError}; /// [`RowIdGenExecutor`] generates row id for data, where the user has not specified a pk. pub struct RowIdGenExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - upstream: Option, + upstream: Option, row_id_index: usize, @@ -45,14 +41,12 @@ pub struct RowIdGenExecutor { impl RowIdGenExecutor { pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - upstream: BoxedExecutor, + upstream: Executor, row_id_index: usize, vnodes: Bitmap, ) -> Self { Self { ctx, - info, upstream: Some(upstream), row_id_index, row_id_generator: Self::new_generator(&vnodes), @@ -126,22 +120,10 @@ impl RowIdGenExecutor { } } -impl Executor for RowIdGenExecutor { +impl Execute for RowIdGenExecutor { fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } #[cfg(test)] @@ -155,7 +137,7 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Executor}; + use crate::executor::{ActorContext, Execute}; #[tokio::test] async fn test_row_id_gen_executor() { @@ -166,20 +148,16 @@ mod tests { let pk_indices = vec![0]; let row_id_index = 0; let row_id_generator = Bitmap::ones(VirtualNode::COUNT); - let (mut tx, upstream) = MockSource::channel(schema.clone(), pk_indices.clone()); - let row_id_gen_executor = Box::new(RowIdGenExecutor::new( - ActorContext::create(233), - ExecutorInfo { - schema, - pk_indices, - identity: "RowIdGenExecutor".to_string(), - }, - Box::new(upstream), + let (mut tx, upstream) = MockSource::channel(); + let upstream = upstream.into_executor(schema.clone(), pk_indices.clone()); + + let row_id_gen_executor = RowIdGenExecutor::new( + ActorContext::for_test(233), + upstream, row_id_index, row_id_generator, - )); - - let mut row_id_gen_executor = row_id_gen_executor.execute(); + ); + let mut row_id_gen_executor = row_id_gen_executor.boxed().execute(); // Init barrier tx.push_barrier(test_epoch(1), false); diff --git a/src/stream/src/executor/simple_agg.rs b/src/stream/src/executor/simple_agg.rs index ddd423adf3e3c..890754f33462f 100644 --- a/src/stream/src/executor/simple_agg.rs +++ b/src/stream/src/executor/simple_agg.rs @@ -47,7 +47,7 @@ use crate::task::AtomicU64Ref; /// Therefore, we "automatically" implemented a window function inside /// `SimpleAggExecutor`. pub struct SimpleAggExecutor { - input: Box, + input: Executor, inner: ExecutorInner, } @@ -111,27 +111,15 @@ struct ExecutionVars { state_changed: bool, } -impl Executor for SimpleAggExecutor { +impl Execute for SimpleAggExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } } impl SimpleAggExecutor { pub fn new(args: AggExecutorArgs) -> StreamResult { - let input_info = args.input.info(); + let input_info = args.input.info().clone(); Ok(Self { input: args.input, inner: ExecutorInner { @@ -214,12 +202,6 @@ impl SimpleAggExecutor { this.intermediate_state_table .update_without_old_value(encoded_states); - // Commit all state tables. - futures::future::try_join_all( - this.all_state_tables_mut().map(|table| table.commit(epoch)), - ) - .await?; - // Retrieve modified states and put the changes into the builders. vars.agg_group .build_change(&this.storages, &this.agg_funcs) @@ -227,13 +209,13 @@ impl SimpleAggExecutor { .map(|change| change.to_stream_chunk(&this.info.schema.data_types())) } else { // No state is changed. - // Call commit on state table to increment the epoch. - this.all_state_tables_mut().for_each(|table| { - table.commit_no_data_expected(epoch); - }); None }; + // Commit all state tables. + futures::future::try_join_all(this.all_state_tables_mut().map(|table| table.commit(epoch))) + .await?; + vars.state_changed = false; Ok(chunk) } @@ -241,7 +223,6 @@ impl SimpleAggExecutor { async fn try_flush_data(this: &mut ExecutorInner) -> StreamExecutorResult<()> { futures::future::try_join_all(this.all_state_tables_mut().map(|table| table.try_flush())) .await?; - Ok(()) } @@ -343,7 +324,8 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![2]); tx.push_barrier(test_epoch(1), false); tx.push_barrier(test_epoch(2), false); tx.push_chunk(StreamChunk::from_pretty( @@ -370,9 +352,9 @@ mod tests { ]; let simple_agg = new_boxed_simple_agg_executor( - ActorContext::create(123), + ActorContext::for_test(123), store, - Box::new(source), + source, false, agg_calls, 0, diff --git a/src/stream/src/executor/sink.rs b/src/stream/src/executor/sink.rs index d33d6b796b891..64c72c053ef34 100644 --- a/src/stream/src/executor/sink.rs +++ b/src/stream/src/executor/sink.rs @@ -34,7 +34,7 @@ use risingwave_connector::sink::{ use thiserror_ext::AsReport; use super::error::{StreamExecutorError, StreamExecutorResult}; -use super::{BoxedExecutor, Executor, ExecutorInfo, Message, PkIndices}; +use super::{Execute, Executor, ExecutorInfo, Message, PkIndices}; use crate::executor::{ expect_first_barrier, ActorContextRef, BoxedMessageStream, MessageStream, Mutation, }; @@ -43,7 +43,7 @@ use crate::task::ActorId; pub struct SinkExecutor { actor_context: ActorContextRef, info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, sink: SinkImpl, input_columns: Vec, sink_param: SinkParam, @@ -83,18 +83,28 @@ impl SinkExecutor { pub async fn new( actor_context: ActorContextRef, info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, sink_writer_param: SinkWriterParam, sink_param: SinkParam, columns: Vec, log_store_factory: F, ) -> StreamExecutorResult { let sink = build_sink(sink_param.clone())?; - let input_schema: Schema = columns + let sink_input_schema: Schema = columns .iter() .map(|column| Field::from(&column.column_desc)) .collect(); - assert_eq!(input_schema.data_types(), info.schema.data_types()); + + if let Some(col_dix) = sink_writer_param.extra_partition_col_idx { + // Remove the partition column from the schema. + assert_eq!(sink_input_schema.data_types(), { + let mut data_type = info.schema.data_types(); + data_type.remove(col_dix); + data_type + }); + } else { + assert_eq!(sink_input_schema.data_types(), info.schema.data_types()); + } Ok(Self { actor_context, @@ -191,6 +201,8 @@ impl SinkExecutor { .init(epoch_pair, barrier.is_pause_on_startup()) .await?; + let mut is_paused = false; + // Propagate the first barrier yield Message::Barrier(barrier); @@ -199,21 +211,29 @@ impl SinkExecutor { match msg? { Message::Watermark(w) => yield Message::Watermark(w), Message::Chunk(chunk) => { + assert!(!is_paused, "Should not receive any data after pause"); log_writer.write_chunk(chunk.clone()).await?; yield Message::Chunk(chunk); } Message::Barrier(barrier) => { + log_writer + .flush_current_epoch(barrier.epoch.curr, barrier.kind.is_checkpoint()) + .await?; + if let Some(mutation) = barrier.mutation.as_deref() { match mutation { - Mutation::Pause => log_writer.pause()?, - Mutation::Resume => log_writer.resume()?, + Mutation::Pause => { + log_writer.pause()?; + is_paused = true; + } + Mutation::Resume => { + log_writer.resume()?; + is_paused = false; + } _ => (), } } - log_writer - .flush_current_epoch(barrier.epoch.curr, barrier.kind.is_checkpoint()) - .await?; if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(actor_id) { log_writer.update_vnode_bitmap(vnode_bitmap).await?; } @@ -402,22 +422,10 @@ impl SinkExecutor { } } -impl Executor for SinkExecutor { +impl Execute for SinkExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } #[cfg(test)] @@ -466,28 +474,25 @@ mod test { .collect(); let pk_indices = vec![0]; - let mock = MockSource::with_messages( - schema.clone(), - pk_indices.clone(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 3 2 1", - ))), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I U- 3 2 1 U+ 3 4 1 + 5 6 7", - ))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I - 5 6 7", - ))), - ], - ); + ))), + ]) + .into_executor(schema.clone(), pk_indices.clone()); let sink_param = SinkParam { sink_id: 0.into(), @@ -511,9 +516,9 @@ mod test { }; let sink_executor = SinkExecutor::new( - ActorContext::create(0), + ActorContext::for_test(0), info, - Box::new(mock), + source, SinkWriterParam::for_test(), sink_param, columns.clone(), @@ -522,7 +527,7 @@ mod test { .await .unwrap(); - let mut executor = SinkExecutor::execute(Box::new(sink_executor)); + let mut executor = sink_executor.boxed().execute(); // Barrier message. executor.next().await.unwrap().unwrap(); @@ -589,32 +594,29 @@ mod test { .map(|column| Field::from(column.column_desc.clone())) .collect(); - let mock = MockSource::with_messages( - schema.clone(), - vec![0, 1], - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 1 1 10", - ))), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 1 3 30", - ))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I + 1 2 20 - 1 2 20", - ))), - Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( - " I I I + ))), + Message::Chunk(std::mem::take(&mut StreamChunk::from_pretty( + " I I I - 1 1 10", - ))), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - ], - ); + ))), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + ]) + .into_executor(schema.clone(), vec![0, 1]); let sink_param = SinkParam { sink_id: 0.into(), @@ -638,9 +640,9 @@ mod test { }; let sink_executor = SinkExecutor::new( - ActorContext::create(0), + ActorContext::for_test(0), info, - Box::new(mock), + source, SinkWriterParam::for_test(), sink_param, columns.clone(), @@ -649,7 +651,7 @@ mod test { .await .unwrap(); - let mut executor = SinkExecutor::execute(Box::new(sink_executor)); + let mut executor = sink_executor.boxed().execute(); // Barrier message. executor.next().await.unwrap().unwrap(); @@ -730,15 +732,12 @@ mod test { .collect(); let pk_indices = vec![0]; - let mock = MockSource::with_messages( - schema.clone(), - pk_indices.clone(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - ], - ); + let source = MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + ]) + .into_executor(schema.clone(), pk_indices.clone()); let sink_param = SinkParam { sink_id: 0.into(), @@ -762,9 +761,9 @@ mod test { }; let sink_executor = SinkExecutor::new( - ActorContext::create(0), + ActorContext::for_test(0), info, - Box::new(mock), + source, SinkWriterParam::for_test(), sink_param, columns, @@ -773,7 +772,7 @@ mod test { .await .unwrap(); - let mut executor = SinkExecutor::execute(Box::new(sink_executor)); + let mut executor = sink_executor.boxed().execute(); // Barrier message. assert_eq!( diff --git a/src/stream/src/executor/sort.rs b/src/stream/src/executor/sort.rs index 95b33edd308c4..dc38d4dcfbeba 100644 --- a/src/stream/src/executor/sort.rs +++ b/src/stream/src/executor/sort.rs @@ -20,23 +20,23 @@ use risingwave_storage::StateStore; use super::sort_buffer::SortBuffer; use super::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, StreamExecutorError, Watermark, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, Watermark, }; use crate::common::table::state_table::StateTable; use crate::common::StreamChunkBuilder; pub struct SortExecutor { - input: BoxedExecutor, + input: Executor, inner: ExecutorInner, } pub struct SortExecutorArgs { pub actor_ctx: ActorContextRef, - pub info: ExecutorInfo, - pub input: BoxedExecutor, + pub input: Executor, + pub schema: Schema, pub buffer_table: StateTable, pub chunk_size: usize, pub sort_column_index: usize, @@ -44,8 +44,8 @@ pub struct SortExecutorArgs { struct ExecutorInner { actor_ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, buffer_table: StateTable, chunk_size: usize, sort_column_index: usize, @@ -53,25 +53,12 @@ struct ExecutorInner { struct ExecutionVars { buffer: SortBuffer, - buffer_changed: bool, } -impl Executor for SortExecutor { +impl Execute for SortExecutor { fn execute(self: Box) -> BoxedMessageStream { self.executor_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.inner.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info.pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info.identity - } } impl SortExecutor { @@ -80,7 +67,7 @@ impl SortExecutor { input: args.input, inner: ExecutorInner { actor_ctx: args.actor_ctx, - info: args.info, + schema: args.schema, buffer_table: args.buffer_table, chunk_size: args.chunk_size, sort_column_index: args.sort_column_index, @@ -103,7 +90,6 @@ impl SortExecutor { let mut vars = ExecutionVars { buffer: SortBuffer::new(this.sort_column_index, &this.buffer_table), - buffer_changed: false, }; // Populate the sort buffer cache on initialization. @@ -116,7 +102,7 @@ impl SortExecutor { if col_idx == this.sort_column_index => { let mut chunk_builder = - StreamChunkBuilder::new(this.chunk_size, this.info.schema.data_types()); + StreamChunkBuilder::new(this.chunk_size, this.schema.data_types()); #[for_await] for row in vars @@ -131,7 +117,6 @@ impl SortExecutor { if let Some(chunk) = chunk_builder.take() { yield Message::Chunk(chunk); } - vars.buffer_changed = true; yield Message::Watermark(watermark); } @@ -141,16 +126,10 @@ impl SortExecutor { } Message::Chunk(chunk) => { vars.buffer.apply_chunk(chunk, &mut this.buffer_table); - vars.buffer_changed = true; this.buffer_table.try_flush().await?; } Message::Barrier(barrier) => { - if vars.buffer_changed { - this.buffer_table.commit(barrier.epoch).await?; - } else { - this.buffer_table.commit_no_data_expected(barrier.epoch); - } - vars.buffer_changed = false; + this.buffer_table.commit(barrier.epoch).await?; // Update the vnode bitmap for state tables of all agg calls if asked. if let Some(vnode_bitmap) = barrier.as_update_vnode_bitmap(this.actor_ctx.id) { @@ -182,7 +161,7 @@ mod tests { use super::*; use crate::executor::test_utils::{MessageSender, MockSource, StreamExecutorTestExt}; - use crate::executor::{ActorContext, BoxedMessageStream, Executor}; + use crate::executor::{ActorContext, BoxedMessageStream, Execute}; async fn create_executor( sort_column_index: usize, @@ -212,15 +191,12 @@ mod tests { ) .await; - let (tx, source) = MockSource::channel(input_schema, input_pk_indices); + let (tx, source) = MockSource::channel(); + let source = source.into_executor(input_schema, input_pk_indices); let sort_executor = SortExecutor::new(SortExecutorArgs { - actor_ctx: ActorContext::create(123), - info: ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "SortExecutor".to_string(), - }, - input: source.boxed(), + actor_ctx: ActorContext::for_test(123), + schema: source.schema().clone(), + input: source, buffer_table, chunk_size: 1024, sort_column_index, diff --git a/src/stream/src/executor/source/executor_core.rs b/src/stream/src/executor/source/executor_core.rs index 82f151bc9ce31..6b3713cc64af1 100644 --- a/src/stream/src/executor/source/executor_core.rs +++ b/src/stream/src/executor/source/executor_core.rs @@ -15,8 +15,8 @@ use std::collections::HashMap; use risingwave_common::catalog::{ColumnId, TableId}; +use risingwave_connector::source::reader::desc::SourceDescBuilder; use risingwave_connector::source::{SplitId, SplitImpl, SplitMetaData}; -use risingwave_source::source_desc::SourceDescBuilder; use risingwave_storage::StateStore; use super::SourceStateTableHandler; @@ -35,13 +35,17 @@ pub struct StreamSourceCore { /// Split info for stream source. A source executor might read data from several splits of /// external connector. - pub(crate) stream_source_splits: HashMap, + pub(crate) latest_split_info: HashMap, /// Stores information of the splits. pub(crate) split_state_store: SourceStateTableHandler, - /// In-memory cache for the splits. - pub(crate) state_cache: HashMap, + /// Contains the latests offsets for the splits that are updated *in the current epoch*. + /// It is cleared after each barrier. + /// + /// Source messages will only write the cache. + /// It is read on split change and rebuild stream reader on error. + pub(crate) updated_splits_in_epoch: HashMap, } impl StreamSourceCore @@ -60,14 +64,14 @@ where source_name, column_ids, source_desc_builder: Some(source_desc_builder), - stream_source_splits: HashMap::new(), + latest_split_info: HashMap::new(), split_state_store, - state_cache: HashMap::new(), + updated_splits_in_epoch: HashMap::new(), } } pub fn init_split_state(&mut self, splits: Vec) { - self.stream_source_splits = splits + self.latest_split_info = splits .into_iter() .map(|split| (split.id(), split)) .collect(); diff --git a/src/stream/src/executor/source/fetch_executor.rs b/src/stream/src/executor/source/fetch_executor.rs index 3aa885cfffe1b..73d1b5b42f1c6 100644 --- a/src/stream/src/executor/source/fetch_executor.rs +++ b/src/stream/src/executor/source/fetch_executor.rs @@ -18,10 +18,11 @@ use std::ops::Bound; use std::sync::Arc; use either::Either; -use futures::pin_mut; use futures::stream::{self, StreamExt}; +use futures::{pin_mut, TryStreamExt}; use futures_async_stream::try_stream; -use risingwave_common::catalog::{ColumnId, Schema, TableId}; +use risingwave_common::array::StreamChunk; +use risingwave_common::catalog::{ColumnId, TableId}; use risingwave_common::hash::VnodeBitmapExt; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{ScalarRef, ScalarRefImpl}; @@ -29,20 +30,20 @@ use risingwave_connector::source::filesystem::opendal_source::{ OpendalGcs, OpendalPosixFs, OpendalS3, OpendalSource, }; use risingwave_connector::source::filesystem::OpendalFsSplit; +use risingwave_connector::source::reader::desc::SourceDesc; use risingwave_connector::source::{ - BoxSourceWithStateStream, SourceContext, SourceCtrlOpts, SplitImpl, SplitMetaData, - StreamChunkWithState, + BoxChunkSourceStream, SourceContext, SourceCtrlOpts, SplitImpl, SplitMetaData, }; use risingwave_connector::ConnectorParams; -use risingwave_source::source_desc::SourceDesc; use risingwave_storage::store::PrefetchOptions; use risingwave_storage::StateStore; use thiserror_ext::AsReport; +use super::{get_split_offset_col_idx, SourceStateTableHandler}; use crate::executor::stream_reader::StreamReaderWithPause; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, Mutation, PkIndicesRef, SourceStateTableHandler, StreamExecutorError, + expect_first_barrier, get_split_offset_mapping_from_chunk, prune_additional_cols, + ActorContextRef, BoxedMessageStream, Execute, Executor, Message, Mutation, StreamExecutorError, StreamExecutorResult, StreamSourceCore, }; @@ -52,13 +53,12 @@ type SplitBatch = Option>; pub struct FsFetchExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: Option>, /// Upstream list executor. - upstream: Option, + upstream: Option, // control options for connector level source_ctrl_opts: SourceCtrlOpts, @@ -73,15 +73,13 @@ impl FsFetchExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: StreamSourceCore, - upstream: BoxedExecutor, + upstream: Executor, source_ctrl_opts: SourceCtrlOpts, connector_params: ConnectorParams, ) -> Self { Self { actor_ctx, - info, stream_source_core: Some(stream_source_core), upstream: Some(upstream), source_ctrl_opts, @@ -96,7 +94,7 @@ impl FsFetchExecutor { column_ids: Vec, source_ctx: SourceContext, source_desc: &SourceDesc, - stream: &mut StreamReaderWithPause, + stream: &mut StreamReaderWithPause, ) -> StreamExecutorResult<()> { let mut batch = Vec::with_capacity(SPLIT_BATCH_SIZE); 'vnodes: for vnode in state_store_handler.state_store.vnodes().iter_vnodes() { @@ -147,7 +145,8 @@ impl FsFetchExecutor { *splits_on_fetch += batch.len(); let batch_reader = Self::build_batched_stream_reader(column_ids, source_ctx, source_desc, Some(batch)) - .await?; + .await? + .map_err(StreamExecutorError::connector_error); stream.replace_data_stream(batch_reader); } @@ -159,15 +158,20 @@ impl FsFetchExecutor { source_ctx: SourceContext, source_desc: &SourceDesc, batch: SplitBatch, - ) -> StreamExecutorResult { + ) -> StreamExecutorResult { source_desc .source - .stream_reader(batch, column_ids, Arc::new(source_ctx)) + .to_stream(batch, column_ids, Arc::new(source_ctx)) .await .map_err(StreamExecutorError::connector_error) } - fn build_source_ctx(&self, source_desc: &SourceDesc, source_id: TableId) -> SourceContext { + fn build_source_ctx( + &self, + source_desc: &SourceDesc, + source_id: TableId, + source_name: &str, + ) -> SourceContext { SourceContext::new_with_suppressor( self.actor_ctx.id, source_id, @@ -177,6 +181,7 @@ impl FsFetchExecutor { self.connector_params.connector_client.clone(), self.actor_ctx.error_suppressor.clone(), source_desc.source.config.clone(), + source_name.to_owned(), ) } @@ -195,14 +200,17 @@ impl FsFetchExecutor { .build() .map_err(StreamExecutorError::connector_error)?; + let (Some(split_idx), Some(offset_idx)) = get_split_offset_col_idx(&source_desc.columns) + else { + unreachable!("Partition and offset columns must be set."); + }; + // Initialize state table. state_store_handler.init_epoch(barrier.epoch); let mut splits_on_fetch: usize = 0; - let mut stream = StreamReaderWithPause::::new( - upstream, - stream::pending().boxed(), - ); + let mut stream = + StreamReaderWithPause::::new(upstream, stream::pending().boxed()); if barrier.is_pause_on_startup() { stream.pause_stream(); @@ -215,7 +223,7 @@ impl FsFetchExecutor { &mut splits_on_fetch, &state_store_handler, core.column_ids.clone(), - self.build_source_ctx(&source_desc, core.source_id), + self.build_source_ctx(&source_desc, core.source_id, &core.source_name), &source_desc, &mut stream, ) @@ -267,7 +275,11 @@ impl FsFetchExecutor { &mut splits_on_fetch, &state_store_handler, core.column_ids.clone(), - self.build_source_ctx(&source_desc, core.source_id), + self.build_source_ctx( + &source_desc, + core.source_id, + &core.source_name, + ), &source_desc, &mut stream, ) @@ -278,7 +290,7 @@ impl FsFetchExecutor { yield msg; } // Receiving file assignments from upstream list executor, - // store into state table and try building a new reader. + // store into state table. Message::Chunk(chunk) => { let file_assignment = chunk .data_chunk() @@ -293,19 +305,17 @@ impl FsFetchExecutor { ) }) .collect(); - state_store_handler.take_snapshot(file_assignment).await?; + state_store_handler.set_states(file_assignment).await?; state_store_handler.state_store.try_flush().await?; } _ => unreachable!(), } } // StreamChunk from FsSourceReader, and the reader reads only one file. - // If the file read out, replace with a new file reader. - Either::Right(StreamChunkWithState { - chunk, - split_offset_mapping, - }) => { - let mapping = split_offset_mapping.unwrap(); + Either::Right(chunk) => { + let mapping = + get_split_offset_mapping_from_chunk(&chunk, split_idx, offset_idx) + .unwrap(); debug_assert_eq!(mapping.len(), 1); if let Some((split_id, offset)) = mapping.into_iter().next() { let row = state_store_handler @@ -331,6 +341,12 @@ impl FsFetchExecutor { } } + let chunk = prune_additional_cols( + &chunk, + split_idx, + offset_idx, + &source_desc.columns, + ); yield Message::Chunk(chunk); } } @@ -340,22 +356,10 @@ impl FsFetchExecutor { } } -impl Executor for FsFetchExecutor { +impl Execute for FsFetchExecutor { fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } impl Debug for FsFetchExecutor { @@ -364,7 +368,6 @@ impl Debug for FsFetchExecutor { f.debug_struct("FsFetchExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("FsFetchExecutor").finish() diff --git a/src/stream/src/executor/source/fs_source_executor.rs b/src/stream/src/executor/source/fs_source_executor.rs index 6275ef5d116f6..d896a1224f6bf 100644 --- a/src/stream/src/executor/source/fs_source_executor.rs +++ b/src/stream/src/executor/source/fs_source_executor.rs @@ -12,22 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. -// *** NOTICE: TO BE DEPRECATED *** // +#![deprecated = "will be replaced by new fs source (list + fetch)"] +#![expect(deprecated)] use std::fmt::{Debug, Formatter}; use std::sync::Arc; use anyhow::anyhow; use either::Either; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; -use risingwave_common::catalog::Schema; use risingwave_common::system_param::local_manager::SystemParamsReaderRef; +use risingwave_common::system_param::reader::SystemParamsRead; +use risingwave_connector::error::ConnectorError; +use risingwave_connector::source::reader::desc::{FsSourceDesc, SourceDescBuilder}; use risingwave_connector::source::{ - BoxSourceWithStateStream, ConnectorState, SourceContext, SourceCtrlOpts, SplitId, SplitImpl, - SplitMetaData, StreamChunkWithState, + BoxChunkSourceStream, ConnectorState, SourceContext, SourceCtrlOpts, SplitId, SplitImpl, + SplitMetaData, }; -use risingwave_source::source_desc::{FsSourceDesc, SourceDescBuilder}; use risingwave_storage::StateStore; use tokio::sync::mpsc::UnboundedReceiver; use tokio::time::Instant; @@ -47,7 +49,6 @@ const WAIT_BARRIER_MULTIPLE_TIMES: u128 = 5; /// such as s3. pub struct FsSourceExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: StreamSourceCore, @@ -68,7 +69,6 @@ impl FsSourceExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: StreamSourceCore, metrics: Arc, barrier_receiver: UnboundedReceiver, @@ -77,7 +77,6 @@ impl FsSourceExecutor { ) -> StreamResult { Ok(Self { actor_ctx, - info, stream_source_core, metrics, barrier_receiver: Some(barrier_receiver), @@ -90,7 +89,7 @@ impl FsSourceExecutor { &mut self, source_desc: &FsSourceDesc, state: ConnectorState, - ) -> StreamExecutorResult { + ) -> StreamExecutorResult { let column_ids = source_desc .columns .iter() @@ -105,10 +104,11 @@ impl FsSourceExecutor { None, self.actor_ctx.error_suppressor.clone(), source_desc.source.config.clone(), + self.stream_source_core.source_name.clone(), ); source_desc .source - .stream_reader(state, column_ids, Arc::new(source_ctx)) + .to_stream(state, column_ids, Arc::new(source_ctx)) .await .map_err(StreamExecutorError::connector_error) } @@ -116,7 +116,7 @@ impl FsSourceExecutor { async fn apply_split_change( &mut self, source_desc: &FsSourceDesc, - stream: &mut StreamReaderWithPause, + stream: &mut StreamReaderWithPause, mapping: &HashMap>, ) -> StreamExecutorResult<()> { if let Some(target_splits) = mapping.get(&self.actor_ctx.id).cloned() { @@ -146,7 +146,7 @@ impl FsSourceExecutor { let mut target_state: Vec = Vec::new(); let mut no_change_flag = true; for sc in rhs { - if let Some(s) = core.state_cache.get(&sc.id()) { + if let Some(s) = core.updated_splits_in_epoch.get(&sc.id()) { let fs = s .as_fs() .unwrap_or_else(|| panic!("split {:?} is not fs", s)); @@ -170,7 +170,7 @@ impl FsSourceExecutor { sc }; - core.state_cache + core.updated_splits_in_epoch .entry(state.id()) .or_insert_with(|| state.clone()); target_state.push(state); @@ -182,7 +182,7 @@ impl FsSourceExecutor { async fn replace_stream_reader_with_target_state( &mut self, source_desc: &FsSourceDesc, - stream: &mut StreamReaderWithPause, + stream: &mut StreamReaderWithPause, target_state: Vec, ) -> StreamExecutorResult<()> { tracing::info!( @@ -194,10 +194,11 @@ impl FsSourceExecutor { // Replace the source reader with a new one of the new state. let reader = self .build_stream_source_reader(source_desc, Some(target_state.clone())) - .await?; + .await? + .map_err(StreamExecutorError::connector_error); stream.replace_data_stream(reader); - self.stream_source_core.stream_source_splits = target_state + self.stream_source_core.latest_split_info = target_state .into_iter() .map(|split| (split.id(), split)) .collect(); @@ -211,7 +212,7 @@ impl FsSourceExecutor { ) -> StreamExecutorResult<()> { let core = &mut self.stream_source_core; let incompleted = core - .state_cache + .updated_splits_in_epoch .values() .filter(|split| { let fs = split @@ -223,7 +224,7 @@ impl FsSourceExecutor { .collect_vec(); let completed = core - .state_cache + .updated_splits_in_epoch .values() .filter(|split| { let fs = split @@ -236,7 +237,7 @@ impl FsSourceExecutor { if !incompleted.is_empty() { tracing::debug!(actor_id = self.actor_ctx.id, incompleted = ?incompleted, "take snapshot"); - core.split_state_store.take_snapshot(incompleted).await? + core.split_state_store.set_states(incompleted).await? } if !completed.is_empty() { @@ -246,7 +247,7 @@ impl FsSourceExecutor { // commit anyway, even if no message saved core.split_state_store.state_store.commit(epoch).await?; - core.state_cache.clear(); + core.updated_splits_in_epoch.clear(); Ok(()) } @@ -280,6 +281,11 @@ impl FsSourceExecutor { .build_fs_source_desc() .map_err(StreamExecutorError::connector_error)?; + let (Some(split_idx), Some(offset_idx)) = get_split_offset_col_idx(&source_desc.columns) + else { + unreachable!("Partition and offset columns must be set."); + }; + // If the first barrier requires us to pause on startup, pause the stream. let start_with_paused = barrier.is_pause_on_startup(); @@ -335,15 +341,14 @@ impl FsSourceExecutor { let source_chunk_reader = self .build_stream_source_reader(&source_desc, recover_state) .instrument_await("fs_source_start_reader") - .await?; + .await? + .map_err(StreamExecutorError::connector_error); // Merge the chunks from source and the barriers into a single stream. We prioritize // barriers over source data chunks here. let barrier_stream = barrier_to_message_stream(barrier_receiver).boxed(); - let mut stream = StreamReaderWithPause::::new( - barrier_stream, - source_chunk_reader, - ); + let mut stream = + StreamReaderWithPause::::new(barrier_stream, source_chunk_reader); if start_with_paused { stream.pause_stream(); } @@ -356,7 +361,6 @@ impl FsSourceExecutor { self.system_params.load().barrier_interval_ms() as u128 * WAIT_BARRIER_MULTIPLE_TIMES; let mut last_barrier_time = Instant::now(); let mut self_paused = false; - let mut metric_row_per_barrier: u64 = 0; while let Some(msg) = stream.next().await { match msg? { // This branch will be preferred. @@ -390,15 +394,6 @@ impl FsSourceExecutor { } self.take_snapshot_and_clear_cache(epoch).await?; - self.metrics - .source_row_per_barrier - .with_label_values(&[ - self.actor_ctx.id.to_string().as_str(), - self.stream_source_core.source_id.to_string().as_ref(), - ]) - .inc_by(metric_row_per_barrier); - metric_row_per_barrier = 0; - yield msg; } _ => { @@ -408,10 +403,10 @@ impl FsSourceExecutor { } }, - Either::Right(StreamChunkWithState { - chunk, - split_offset_mapping, - }) => { + Either::Right(chunk) => { + // TODO: confirm when split_offset_mapping is None + let split_offset_mapping = + get_split_offset_mapping_from_chunk(&chunk, split_idx, offset_idx); if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms { // Exceeds the max wait barrier time, the source will be paused. Currently // we can guarantee the source is not paused since it received stream @@ -430,17 +425,18 @@ impl FsSourceExecutor { let state: Vec<(SplitId, SplitImpl)> = mapping .iter() .flat_map(|(id, offset)| { - let origin_split = - self.stream_source_core.stream_source_splits.get_mut(id); - - origin_split.map(|split| { - split.update_in_place(offset.clone())?; - Ok::<_, anyhow::Error>((id.clone(), split.clone())) - }) + self.stream_source_core.latest_split_info.get_mut(id).map( + |origin_split| { + origin_split.update_in_place(offset.clone())?; + Ok::<_, ConnectorError>((id.clone(), origin_split.clone())) + }, + ) }) .try_collect()?; - self.stream_source_core.state_cache.extend(state); + self.stream_source_core + .updated_splits_in_epoch + .extend(state); } self.metrics @@ -449,9 +445,14 @@ impl FsSourceExecutor { self.stream_source_core.source_id.to_string().as_ref(), self.stream_source_core.source_name.as_ref(), self.actor_ctx.id.to_string().as_str(), + self.actor_ctx.fragment_id.to_string().as_str(), ]) .inc_by(chunk.cardinality() as u64); + + let chunk = + prune_additional_cols(&chunk, split_idx, offset_idx, &source_desc.columns); yield Message::Chunk(chunk); + self.try_flush_data().await?; } } @@ -465,22 +466,10 @@ impl FsSourceExecutor { } } -impl Executor for FsSourceExecutor { +impl Execute for FsSourceExecutor { fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } impl Debug for FsSourceExecutor { @@ -488,7 +477,6 @@ impl Debug for FsSourceExecutor { f.debug_struct("FsSourceExecutor") .field("source_id", &self.stream_source_core.source_id) .field("column_ids", &self.stream_source_core.column_ids) - .field("pk_indices", &self.info.pk_indices) .finish() } } diff --git a/src/stream/src/executor/source/list_executor.rs b/src/stream/src/executor/source/list_executor.rs index 51dc1de6b7069..7996848a749e3 100644 --- a/src/stream/src/executor/source/list_executor.rs +++ b/src/stream/src/executor/source/list_executor.rs @@ -17,15 +17,13 @@ use std::sync::Arc; use anyhow::anyhow; use either::Either; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use risingwave_common::array::Op; -use risingwave_common::catalog::Schema; use risingwave_common::system_param::local_manager::SystemParamsReaderRef; -use risingwave_connector::source::filesystem::FsPageItem; -use risingwave_connector::source::{BoxTryStream, SourceCtrlOpts}; +use risingwave_connector::source::reader::desc::{SourceDesc, SourceDescBuilder}; +use risingwave_connector::source::SourceCtrlOpts; use risingwave_connector::ConnectorParams; -use risingwave_source::source_desc::{SourceDesc, SourceDescBuilder}; use risingwave_storage::StateStore; use thiserror_ext::AsReport; use tokio::sync::mpsc::UnboundedReceiver; @@ -40,7 +38,6 @@ const CHUNK_SIZE: usize = 1024; #[allow(dead_code)] pub struct FsListExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: Option>, @@ -65,7 +62,6 @@ impl FsListExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: Option>, metrics: Arc, barrier_receiver: UnboundedReceiver, @@ -75,7 +71,6 @@ impl FsListExecutor { ) -> Self { Self { actor_ctx, - info, stream_source_core, metrics, barrier_receiver: Some(barrier_receiver), @@ -89,13 +84,12 @@ impl FsListExecutor { fn build_chunked_paginate_stream( &self, source_desc: &SourceDesc, - ) -> StreamExecutorResult> { - let stream: std::pin::Pin< - Box> + Send>, - > = source_desc + ) -> StreamExecutorResult>> { + let stream = source_desc .source .get_source_list() - .map_err(StreamExecutorError::connector_error)?; + .map_err(StreamExecutorError::connector_error)? + .map_err(StreamExecutorError::connector_error); // Group FsPageItem stream into chunks of size 1024. let chunked_stream = stream.chunks(CHUNK_SIZE).map(|chunk| { @@ -120,7 +114,7 @@ impl FsListExecutor { )) }); - Ok(chunked_stream.boxed()) + Ok(chunked_stream) } #[try_stream(ok = Message, error = StreamExecutorError)] @@ -195,22 +189,10 @@ impl FsListExecutor { } } -impl Executor for FsListExecutor { +impl Execute for FsListExecutor { fn execute(self: Box) -> BoxedMessageStream { self.into_stream().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } impl Debug for FsListExecutor { @@ -219,7 +201,6 @@ impl Debug for FsListExecutor { f.debug_struct("FsListExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("FsListExecutor").finish() diff --git a/src/stream/src/executor/source/mod.rs b/src/stream/src/executor/source/mod.rs index 318581bb996f6..8cdba698bec12 100644 --- a/src/stream/src/executor/source/mod.rs +++ b/src/stream/src/executor/source/mod.rs @@ -13,11 +13,20 @@ // limitations under the License. pub mod executor_core; +use std::collections::HashMap; + use await_tree::InstrumentAwait; pub use executor_core::StreamSourceCore; mod fs_source_executor; +#[expect(deprecated)] pub use fs_source_executor::*; +use itertools::Itertools; +use risingwave_common::array::StreamChunk; use risingwave_common::bail; +use risingwave_common::row::Row; +use risingwave_connector::source::{SourceColumnDesc, SplitId}; +use risingwave_pb::plan_common::additional_column::ColumnType; +use risingwave_pb::plan_common::AdditionalColumn; pub use state_table_handler::*; pub mod fetch_executor; pub use fetch_executor::*; @@ -41,3 +50,55 @@ pub async fn barrier_to_message_stream(mut rx: UnboundedReceiver) { } bail!("barrier reader closed unexpectedly"); } + +pub fn get_split_offset_mapping_from_chunk( + chunk: &StreamChunk, + split_idx: usize, + offset_idx: usize, +) -> Option> { + let mut split_offset_mapping = HashMap::new(); + for (_, row) in chunk.rows() { + let split_id = row.datum_at(split_idx).unwrap().into_utf8().into(); + let offset = row.datum_at(offset_idx).unwrap().into_utf8(); + split_offset_mapping.insert(split_id, offset.to_string()); + } + Some(split_offset_mapping) +} + +pub fn get_split_offset_col_idx( + column_descs: &[SourceColumnDesc], +) -> (Option, Option) { + let mut split_idx = None; + let mut offset_idx = None; + for (idx, column) in column_descs.iter().enumerate() { + match column.additional_column { + AdditionalColumn { + column_type: Some(ColumnType::Partition(_) | ColumnType::Filename(_)), + } => { + split_idx = Some(idx); + } + AdditionalColumn { + column_type: Some(ColumnType::Offset(_)), + } => { + offset_idx = Some(idx); + } + _ => (), + } + } + (split_idx, offset_idx) +} + +pub fn prune_additional_cols( + chunk: &StreamChunk, + split_idx: usize, + offset_idx: usize, + column_descs: &[SourceColumnDesc], +) -> StreamChunk { + chunk.project( + &(0..chunk.dimension()) + .filter(|&idx| { + (idx != split_idx && idx != offset_idx) || column_descs[idx].is_visible() + }) + .collect_vec(), + ) +} diff --git a/src/stream/src/executor/source/source_executor.rs b/src/stream/src/executor/source/source_executor.rs index 0f3b3aaf34028..ca89970f7dec5 100644 --- a/src/stream/src/executor/source/source_executor.rs +++ b/src/stream/src/executor/source/source_executor.rs @@ -17,16 +17,16 @@ use std::time::Duration; use anyhow::anyhow; use either::Either; -use futures::StreamExt; +use futures::{StreamExt, TryStreamExt}; use futures_async_stream::try_stream; use risingwave_common::metrics::GLOBAL_ERROR_METRICS; use risingwave_common::system_param::local_manager::SystemParamsReaderRef; +use risingwave_common::system_param::reader::SystemParamsRead; +use risingwave_connector::source::reader::desc::{SourceDesc, SourceDescBuilder}; use risingwave_connector::source::{ - BoxSourceWithStateStream, ConnectorState, SourceContext, SourceCtrlOpts, SplitMetaData, - StreamChunkWithState, + BoxChunkSourceStream, ConnectorState, SourceContext, SourceCtrlOpts, SplitId, SplitMetaData, }; use risingwave_connector::ConnectorParams; -use risingwave_source::source_desc::{SourceDesc, SourceDescBuilder}; use risingwave_storage::StateStore; use thiserror_ext::AsReport; use tokio::sync::mpsc::UnboundedReceiver; @@ -43,7 +43,6 @@ const WAIT_BARRIER_MULTIPLE_TIMES: u128 = 5; pub struct SourceExecutor { actor_ctx: ActorContextRef, - info: ExecutorInfo, /// Streaming source for external stream_source_core: Option>, @@ -68,7 +67,6 @@ impl SourceExecutor { #[allow(clippy::too_many_arguments)] pub fn new( actor_ctx: ActorContextRef, - info: ExecutorInfo, stream_source_core: Option>, metrics: Arc, barrier_receiver: UnboundedReceiver, @@ -78,7 +76,6 @@ impl SourceExecutor { ) -> Self { Self { actor_ctx, - info, stream_source_core, metrics, barrier_receiver: Some(barrier_receiver), @@ -92,7 +89,7 @@ impl SourceExecutor { &self, source_desc: &SourceDesc, state: ConnectorState, - ) -> StreamExecutorResult { + ) -> StreamExecutorResult { let column_ids = source_desc .columns .iter() @@ -107,16 +104,21 @@ impl SourceExecutor { self.connector_params.connector_client.clone(), self.actor_ctx.error_suppressor.clone(), source_desc.source.config.clone(), + self.stream_source_core + .as_ref() + .unwrap() + .source_name + .clone(), ); source_desc .source - .stream_reader(state, column_ids, Arc::new(source_ctx)) + .to_stream(state, column_ids, Arc::new(source_ctx)) .await .map_err(StreamExecutorError::connector_error) } #[inline] - fn get_metric_labels(&self) -> [String; 3] { + fn get_metric_labels(&self) -> [String; 4] { [ self.stream_source_core .as_ref() @@ -129,15 +131,25 @@ impl SourceExecutor { .source_name .clone(), self.actor_ctx.id.to_string(), + self.actor_ctx.fragment_id.to_string(), ] } + /// - `should_trim_state`: whether to trim state for dropped splits. + /// + /// For scaling, the connector splits can be migrated to other actors, but + /// won't be added or removed. Actors should not trim states for splits that + /// are moved to other actors. + /// + /// For source split change, split will not be migrated and we can trim states + /// for deleted splits. async fn apply_split_change( &mut self, source_desc: &SourceDesc, - stream: &mut StreamReaderWithPause, + stream: &mut StreamReaderWithPause, split_assignment: &HashMap>, - ) -> StreamExecutorResult>> { + should_trim_state: bool, + ) -> StreamExecutorResult<()> { self.metrics .source_split_change_count .with_label_values( @@ -149,94 +161,111 @@ impl SourceExecutor { ) .inc(); if let Some(target_splits) = split_assignment.get(&self.actor_ctx.id).cloned() { - if let Some(target_state) = self.update_state_if_changed(Some(target_splits)).await? { - tracing::info!( - actor_id = self.actor_ctx.id, - state = ?target_state, - "apply split change" - ); - - self.replace_stream_reader_with_target_state( - source_desc, - stream, - target_state.clone(), - ) - .await?; - - return Ok(Some(target_state)); + if self + .update_state_if_changed(target_splits, should_trim_state) + .await? + { + self.rebuild_stream_reader(source_desc, stream).await?; } } - Ok(None) + Ok(()) } - // Note: `update_state_if_changed` will modify `state_cache` + /// Returns `true` if split changed. Otherwise `false`. async fn update_state_if_changed( &mut self, - state: ConnectorState, - ) -> StreamExecutorResult { + target_splits: Vec, + should_trim_state: bool, + ) -> StreamExecutorResult { let core = self.stream_source_core.as_mut().unwrap(); - let target_splits: HashMap<_, _> = state - .unwrap() + let target_splits: HashMap<_, _> = target_splits .into_iter() .map(|split| (split.id(), split)) .collect(); - let mut target_state: Vec = Vec::with_capacity(target_splits.len()); + let mut target_state: HashMap = + HashMap::with_capacity(target_splits.len()); let mut split_changed = false; - for (split_id, split) in &target_splits { - if let Some(s) = core.state_cache.get(split_id) { - // existing split, no change, clone from cache - target_state.push(s.clone()) + // Checks added splits + for (split_id, split) in target_splits { + if let Some(s) = core.latest_split_info.get(&split_id) { + // For existing splits, we should use the latest offset from the cache. + // `target_splits` is from meta and contains the initial offset. + target_state.insert(split_id, s.clone()); } else { split_changed = true; // write new assigned split to state cache. snapshot is base on cache. let initial_state = if let Some(recover_state) = core .split_state_store - .try_recover_from_state_store(split) + .try_recover_from_state_store(&split) .await? { recover_state } else { - split.clone() + split }; - core.state_cache - .entry(split.id()) + core.updated_splits_in_epoch + .entry(split_id.clone()) .or_insert_with(|| initial_state.clone()); - target_state.push(initial_state); + target_state.insert(split_id, initial_state); } } - // state cache may be stale - for existing_split_id in core.stream_source_splits.keys() { - if !target_splits.contains_key(existing_split_id) { + // Checks dropped splits + for existing_split_id in core.latest_split_info.keys() { + if !target_state.contains_key(existing_split_id) { tracing::info!("split dropping detected: {}", existing_split_id); split_changed = true; } } - Ok(split_changed.then_some(target_state)) + if split_changed { + tracing::info!( + actor_id = self.actor_ctx.id, + state = ?target_state, + "apply split change" + ); + + core.updated_splits_in_epoch + .retain(|split_id, _| target_state.get(split_id).is_some()); + + let dropped_splits = core + .latest_split_info + .extract_if(|split_id, _| target_state.get(split_id).is_none()) + .map(|(_, split)| split) + .collect_vec(); + + if should_trim_state && !dropped_splits.is_empty() { + // trim dropped splits' state + core.split_state_store.trim_state(&dropped_splits).await?; + } + + core.latest_split_info = target_state; + } + + Ok(split_changed) } /// Rebuild stream if there is a err in stream async fn rebuild_stream_reader_from_error( &mut self, source_desc: &SourceDesc, - stream: &mut StreamReaderWithPause, - split_info: &mut [SplitImpl], + stream: &mut StreamReaderWithPause, e: StreamExecutorError, ) -> StreamExecutorResult<()> { let core = self.stream_source_core.as_mut().unwrap(); tracing::warn!( - "stream source reader error, actor: {:?}, source: {:?}", - self.actor_ctx.id, - core.source_id, + error = ?e.as_report(), + actor_id = self.actor_ctx.id, + source_id = %core.source_id, + "stream source reader error", ); GLOBAL_ERROR_METRICS.user_source_reader_error.report([ "SourceReaderError".to_owned(), @@ -245,36 +274,18 @@ impl SourceExecutor { self.actor_ctx.id.to_string(), core.source_id.to_string(), ]); - // fetch the newest offset, either it's in cache (before barrier) - // or in state table (just after barrier) - let target_state = if core.state_cache.is_empty() { - for ele in &mut *split_info { - if let Some(recover_state) = core - .split_state_store - .try_recover_from_state_store(ele) - .await? - { - *ele = recover_state; - } - } - split_info.to_owned() - } else { - core.state_cache - .values() - .map(|split_impl| split_impl.to_owned()) - .collect_vec() - }; - self.replace_stream_reader_with_target_state(source_desc, stream, target_state) - .await + self.rebuild_stream_reader(source_desc, stream).await } - async fn replace_stream_reader_with_target_state( + async fn rebuild_stream_reader( &mut self, source_desc: &SourceDesc, - stream: &mut StreamReaderWithPause, - target_state: Vec, + stream: &mut StreamReaderWithPause, ) -> StreamExecutorResult<()> { + let core = self.stream_source_core.as_mut().unwrap(); + let target_state: Vec = core.latest_split_info.values().cloned().collect(); + tracing::info!( "actor {:?} apply source split change to {:?}", self.actor_ctx.id, @@ -284,62 +295,39 @@ impl SourceExecutor { // Replace the source reader with a new one of the new state. let reader = self .build_stream_source_reader(source_desc, Some(target_state.clone())) - .await?; + .await? + .map_err(StreamExecutorError::connector_error); stream.replace_data_stream(reader); Ok(()) } - async fn take_snapshot_and_clear_cache( + async fn persist_state_and_clear_cache( &mut self, epoch: EpochPair, - target_state: Option>, - should_trim_state: bool, ) -> StreamExecutorResult<()> { let core = self.stream_source_core.as_mut().unwrap(); - let mut cache = core - .state_cache + let cache = core + .updated_splits_in_epoch .values() .map(|split_impl| split_impl.to_owned()) .collect_vec(); - if let Some(target_splits) = target_state { - let target_split_ids: HashSet<_> = - target_splits.iter().map(|split| split.id()).collect(); - - cache.retain(|split| target_split_ids.contains(&split.id())); - - let dropped_splits = core - .stream_source_splits - .extract_if(|split_id, _| !target_split_ids.contains(split_id)) - .map(|(_, split)| split) - .collect_vec(); - - if should_trim_state && !dropped_splits.is_empty() { - // trim dropped splits' state - core.split_state_store.trim_state(&dropped_splits).await?; - } - - core.stream_source_splits = target_splits - .into_iter() - .map(|split| (split.id(), split)) - .collect(); - } - if !cache.is_empty() { tracing::debug!(actor_id = self.actor_ctx.id, state = ?cache, "take snapshot"); - core.split_state_store.take_snapshot(cache).await? + core.split_state_store.set_states(cache).await?; } + // commit anyway, even if no message saved core.split_state_store.state_store.commit(epoch).await?; - - core.state_cache.clear(); + core.updated_splits_in_epoch.clear(); Ok(()) } + /// try mem table spill async fn try_flush_data(&mut self) -> StreamExecutorResult<()> { let core = self.stream_source_core.as_mut().unwrap(); core.split_state_store.state_store.try_flush().await?; @@ -374,9 +362,14 @@ impl SourceExecutor { .build() .map_err(StreamExecutorError::connector_error)?; + let (Some(split_idx), Some(offset_idx)) = get_split_offset_col_idx(&source_desc.columns) + else { + unreachable!("Partition and offset columns must be set."); + }; + let mut boot_state = Vec::default(); - if let Some(mutation) = barrier.mutation.as_ref() { - match mutation.as_ref() { + if let Some(mutation) = barrier.mutation.as_deref() { + match mutation { Mutation::Add(AddMutation { splits, .. }) | Mutation::Update(UpdateMutation { actor_splits: splits, @@ -394,7 +387,6 @@ impl SourceExecutor { _ => {} } } - let mut latest_split_info = boot_state.clone(); core.split_state_store.init_epoch(barrier.epoch); @@ -419,15 +411,14 @@ impl SourceExecutor { let source_chunk_reader = self .build_stream_source_reader(&source_desc, recover_state) .instrument_await("source_build_reader") - .await?; + .await? + .map_err(StreamExecutorError::connector_error); // Merge the chunks from source and the barriers into a single stream. We prioritize // barriers over source data chunks here. let barrier_stream = barrier_to_message_stream(barrier_receiver).boxed(); - let mut stream = StreamReaderWithPause::::new( - barrier_stream, - source_chunk_reader, - ); + let mut stream = + StreamReaderWithPause::::new(barrier_stream, source_chunk_reader); // If the first barrier requires us to pause on startup, pause the stream. if barrier.is_pause_on_startup() { @@ -442,175 +433,134 @@ impl SourceExecutor { self.system_params.load().barrier_interval_ms() as u128 * WAIT_BARRIER_MULTIPLE_TIMES; let mut last_barrier_time = Instant::now(); let mut self_paused = false; - let mut metric_row_per_barrier: u64 = 0; while let Some(msg) = stream.next().await { - match msg { - Err(e) => { - tokio::time::sleep(Duration::from_millis(1000)).await; - self.rebuild_stream_reader_from_error( - &source_desc, - &mut stream, - &mut latest_split_info, - e, - ) + let Ok(msg) = msg else { + tokio::time::sleep(Duration::from_millis(1000)).await; + self.rebuild_stream_reader_from_error(&source_desc, &mut stream, msg.unwrap_err()) .await?; - } - Ok(msg) => { - match msg { - // This branch will be preferred. - Either::Left(msg) => match &msg { - Message::Barrier(barrier) => { - last_barrier_time = Instant::now(); - - if self_paused { - stream.resume_stream(); - self_paused = false; - } - - let epoch = barrier.epoch; - - let mut target_state = None; - let mut should_trim_state = false; - - if let Some(ref mutation) = barrier.mutation.as_deref() { - match mutation { - Mutation::Pause => stream.pause_stream(), - Mutation::Resume => stream.resume_stream(), - Mutation::SourceChangeSplit(actor_splits) => { - tracing::info!( - actor_id = self.actor_ctx.id, - actor_splits = ?actor_splits, - "source change split received" - ); - - target_state = self - .apply_split_change( - &source_desc, - &mut stream, - actor_splits, - ) - .await?; - should_trim_state = true; - } - - Mutation::Update(UpdateMutation { - actor_splits, .. - }) => { - target_state = self - .apply_split_change( - &source_desc, - &mut stream, - actor_splits, - ) - .await?; - } - _ => {} - } - } - - if let Some(target_state) = &target_state { - latest_split_info = target_state.clone(); - } - - self.take_snapshot_and_clear_cache( - epoch, - target_state, - should_trim_state, + continue; + }; + + match msg { + // This branch will be preferred. + Either::Left(Message::Barrier(barrier)) => { + last_barrier_time = Instant::now(); + + if self_paused { + stream.resume_stream(); + self_paused = false; + } + + let epoch = barrier.epoch; + + if let Some(mutation) = barrier.mutation.as_deref() { + match mutation { + Mutation::Pause => stream.pause_stream(), + Mutation::Resume => stream.resume_stream(), + Mutation::SourceChangeSplit(actor_splits) => { + tracing::info!( + actor_id = self.actor_ctx.id, + actor_splits = ?actor_splits, + "source change split received" + ); + + self.apply_split_change( + &source_desc, + &mut stream, + actor_splits, + true, ) .await?; - - self.metrics - .source_row_per_barrier - .with_label_values(&[ - self.actor_ctx.id.to_string().as_str(), - self.stream_source_core - .as_ref() - .unwrap() - .source_id - .to_string() - .as_ref(), - ]) - .inc_by(metric_row_per_barrier); - metric_row_per_barrier = 0; - - yield msg; - } - _ => { - // For the source executor, the message we receive from this arm - // should always be barrier message. - unreachable!(); } - }, - - Either::Right(StreamChunkWithState { - chunk, - split_offset_mapping, - }) => { - if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms { - // Exceeds the max wait barrier time, the source will be paused. - // Currently we can guarantee the - // source is not paused since it received stream - // chunks. - self_paused = true; - tracing::warn!( - "source {} paused, wait barrier for {:?}", - self.info.identity, - last_barrier_time.elapsed() - ); - stream.pause_stream(); - - // Only update `max_wait_barrier_time_ms` to capture - // `barrier_interval_ms` - // changes here to avoid frequently accessing the shared - // `system_params`. - max_wait_barrier_time_ms = - self.system_params.load().barrier_interval_ms() as u128 - * WAIT_BARRIER_MULTIPLE_TIMES; + + Mutation::Update(UpdateMutation { actor_splits, .. }) => { + self.apply_split_change( + &source_desc, + &mut stream, + actor_splits, + false, + ) + .await?; } - if let Some(mapping) = split_offset_mapping { - let state: HashMap<_, _> = mapping - .iter() - .flat_map(|(split_id, offset)| { - let origin_split_impl = self - .stream_source_core - .as_mut() - .unwrap() - .stream_source_splits - .get_mut(split_id); - - origin_split_impl.map(|split_impl| { - split_impl.update_in_place(offset.clone())?; - Ok::<_, anyhow::Error>(( - split_id.clone(), - split_impl.clone(), - )) - }) - }) - .try_collect()?; + _ => {} + } + } + self.persist_state_and_clear_cache(epoch).await?; + + yield Message::Barrier(barrier); + } + Either::Left(_) => { + // For the source executor, the message we receive from this arm + // should always be barrier message. + unreachable!(); + } + + Either::Right(chunk) => { + // TODO: confirm when split_offset_mapping is None + let split_offset_mapping = + get_split_offset_mapping_from_chunk(&chunk, split_idx, offset_idx); + if last_barrier_time.elapsed().as_millis() > max_wait_barrier_time_ms { + // Exceeds the max wait barrier time, the source will be paused. + // Currently we can guarantee the + // source is not paused since it received stream + // chunks. + self_paused = true; + tracing::warn!( + "source paused, wait barrier for {:?}", + last_barrier_time.elapsed() + ); + stream.pause_stream(); + + // Only update `max_wait_barrier_time_ms` to capture + // `barrier_interval_ms` + // changes here to avoid frequently accessing the shared + // `system_params`. + max_wait_barrier_time_ms = self.system_params.load().barrier_interval_ms() + as u128 + * WAIT_BARRIER_MULTIPLE_TIMES; + } + if let Some(mapping) = split_offset_mapping { + let state: HashMap<_, _> = mapping + .iter() + .flat_map(|(split_id, offset)| { self.stream_source_core .as_mut() .unwrap() - .state_cache - .extend(state); - } - metric_row_per_barrier += chunk.cardinality() as u64; - - self.metrics - .source_output_row_count - .with_label_values( - &self - .get_metric_labels() - .iter() - .map(AsRef::as_ref) - .collect::>(), - ) - .inc_by(chunk.cardinality() as u64); - yield Message::Chunk(chunk); - self.try_flush_data().await?; - } + .latest_split_info + .get_mut(split_id) + .map(|original_split_impl| { + original_split_impl.update_in_place(offset.clone())?; + Ok::<_, anyhow::Error>(( + split_id.clone(), + original_split_impl.clone(), + )) + }) + }) + .try_collect()?; + + self.stream_source_core + .as_mut() + .unwrap() + .updated_splits_in_epoch + .extend(state); } + + self.metrics + .source_output_row_count + .with_label_values( + &self + .get_metric_labels() + .iter() + .map(AsRef::as_ref) + .collect::>(), + ) + .inc_by(chunk.cardinality() as u64); + let chunk = + prune_additional_cols(&chunk, split_idx, offset_idx, &source_desc.columns); + yield Message::Chunk(chunk); + self.try_flush_data().await?; } } } @@ -645,7 +595,7 @@ impl SourceExecutor { } } -impl Executor for SourceExecutor { +impl Execute for SourceExecutor { fn execute(self: Box) -> BoxedMessageStream { if self.stream_source_core.is_some() { self.execute_with_stream_source().boxed() @@ -653,18 +603,6 @@ impl Executor for SourceExecutor { self.execute_without_stream_source().boxed() } } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } impl Debug for SourceExecutor { @@ -673,7 +611,6 @@ impl Debug for SourceExecutor { f.debug_struct("SourceExecutor") .field("source_id", &core.source_id) .field("column_ids", &core.column_ids) - .field("pk_indices", &self.info.pk_indices) .finish() } else { f.debug_struct("SourceExecutor").finish() @@ -694,9 +631,9 @@ mod tests { use risingwave_common::types::DataType; use risingwave_common::util::epoch::test_epoch; use risingwave_connector::source::datagen::DatagenSplit; + use risingwave_connector::source::reader::desc::test_utils::create_source_desc_builder; use risingwave_pb::catalog::StreamSourceInfo; use risingwave_pb::plan_common::PbRowFormatType; - use risingwave_source::connector_test_utils::create_source_desc_builder; use risingwave_storage::memory::MemoryStateStore; use tokio::sync::mpsc::unbounded_channel; use tracing_test::traced_test; @@ -713,7 +650,6 @@ mod tests { fields: vec![Field::with_name(DataType::Int32, "sequence_int")], }; let row_id_index = None; - let pk_indices = vec![0]; let source_info = StreamSourceInfo { row_format: PbRowFormatType::Native as i32, ..Default::default() @@ -740,21 +676,16 @@ mod tests { source_id: table_id, column_ids, source_desc_builder: Some(source_desc_builder), - stream_source_splits: HashMap::new(), + latest_split_info: HashMap::new(), split_state_store, - state_cache: HashMap::new(), + updated_splits_in_epoch: HashMap::new(), source_name: MOCK_SOURCE_NAME.to_string(), }; let system_params_manager = LocalSystemParamsManager::for_test(); let executor = SourceExecutor::new( - ActorContext::create(0), - ExecutorInfo { - schema, - pk_indices, - identity: "SourceExecutor".to_string(), - }, + ActorContext::for_test(0), Some(core), Arc::new(StreamingMetrics::unused()), barrier_rx, @@ -762,7 +693,7 @@ mod tests { SourceCtrlOpts::default(), ConnectorParams::default(), ); - let mut executor = Box::new(executor).execute(); + let mut executor = executor.boxed().execute(); let init_barrier = Barrier::new_test_barrier(test_epoch(1)).with_mutation(Mutation::Add(AddMutation { @@ -807,7 +738,6 @@ mod tests { fields: vec![Field::with_name(DataType::Int32, "v1")], }; let row_id_index = None; - let pk_indices = vec![0_usize]; let source_info = StreamSourceInfo { row_format: PbRowFormatType::Native as i32, ..Default::default() @@ -835,21 +765,16 @@ mod tests { source_id: table_id, column_ids: column_ids.clone(), source_desc_builder: Some(source_desc_builder), - stream_source_splits: HashMap::new(), + latest_split_info: HashMap::new(), split_state_store, - state_cache: HashMap::new(), + updated_splits_in_epoch: HashMap::new(), source_name: MOCK_SOURCE_NAME.to_string(), }; let system_params_manager = LocalSystemParamsManager::for_test(); let executor = SourceExecutor::new( - ActorContext::create(0), - ExecutorInfo { - schema, - pk_indices, - identity: "SourceExecutor".to_string(), - }, + ActorContext::for_test(0), Some(core), Arc::new(StreamingMetrics::unused()), barrier_rx, @@ -857,7 +782,7 @@ mod tests { SourceCtrlOpts::default(), ConnectorParams::default(), ); - let mut handler = Box::new(executor).execute(); + let mut handler = executor.boxed().execute(); let init_barrier = Barrier::new_test_barrier(test_epoch(1)).with_mutation(Mutation::Add(AddMutation { diff --git a/src/stream/src/executor/source/state_table_handler.rs b/src/stream/src/executor/source/state_table_handler.rs index b42f3fc9dc60b..c9d967ca56c8f 100644 --- a/src/stream/src/executor/source/state_table_handler.rs +++ b/src/stream/src/executor/source/state_table_handler.rs @@ -12,14 +12,23 @@ // See the License for the specific language governing permissions and // limitations under the License. +cfg_if::cfg_if! { + if #[cfg(test)] { + use risingwave_common::catalog::{DatabaseId, SchemaId}; + use risingwave_pb::catalog::table::TableType; + use risingwave_pb::common::{PbColumnOrder, PbDirection, PbNullsAre, PbOrderType}; + use risingwave_pb::data::data_type::TypeName; + use risingwave_pb::data::DataType; + use risingwave_pb::plan_common::{ColumnCatalog, ColumnDesc}; + } +} + use std::collections::HashSet; use std::ops::{Bound, Deref}; use std::sync::Arc; use futures::{pin_mut, StreamExt}; use risingwave_common::buffer::Bitmap; -use risingwave_common::catalog::{DatabaseId, SchemaId}; -use risingwave_common::constants::hummock::PROPERTIES_RETENTION_SECOND_KEY; use risingwave_common::hash::VirtualNode; use risingwave_common::row::{OwnedRow, Row}; use risingwave_common::types::{JsonbVal, ScalarImpl, ScalarRef, ScalarRefImpl}; @@ -27,12 +36,7 @@ use risingwave_common::util::epoch::EpochPair; use risingwave_common::{bail, row}; use risingwave_connector::source::{SplitId, SplitImpl, SplitMetaData}; use risingwave_hummock_sdk::key::next_key; -use risingwave_pb::catalog::table::TableType; use risingwave_pb::catalog::PbTable; -use risingwave_pb::common::{PbColumnOrder, PbDirection, PbNullsAre, PbOrderType}; -use risingwave_pb::data::data_type::TypeName; -use risingwave_pb::data::DataType; -use risingwave_pb::plan_common::{ColumnCatalog, ColumnDesc}; use risingwave_storage::store::PrefetchOptions; use risingwave_storage::StateStore; @@ -48,11 +52,6 @@ pub struct SourceStateTableHandler { impl SourceStateTableHandler { pub async fn from_table_catalog(table_catalog: &PbTable, store: S) -> Self { - // The state of source should not be cleaned up by retention_seconds - assert!(!table_catalog - .properties - .contains_key(&String::from(PROPERTIES_RETENTION_SECOND_KEY))); - Self { state_store: StateTable::from_table_catalog(table_catalog, store, None).await, } @@ -63,11 +62,6 @@ impl SourceStateTableHandler { store: S, vnodes: Option>, ) -> Self { - // The state of source should not be cleaned up by retention_seconds - assert!(!table_catalog - .properties - .contains_key(&String::from(PROPERTIES_RETENTION_SECOND_KEY))); - Self { state_store: StateTable::from_table_catalog(table_catalog, store, vnodes).await, } @@ -140,10 +134,10 @@ impl SourceStateTableHandler { /// set all complete /// can only used by `FsSourceExecutor` - pub(crate) async fn set_all_complete(&mut self, states: Vec) -> StreamExecutorResult<()> - where - SS: SplitMetaData, - { + pub(crate) async fn set_all_complete( + &mut self, + states: Vec, + ) -> StreamExecutorResult<()> { if states.is_empty() { // TODO should be a clear Error Code bail!("states require not null"); @@ -180,30 +174,18 @@ impl SourceStateTableHandler { Ok(()) } - /// This function provides the ability to persist the source state - /// and needs to be invoked by the ``SourceReader`` to call it, - /// and will return the error when the dependent ``StateStore`` handles the error. - /// The caller should ensure that the passed parameters are not empty. - pub async fn take_snapshot(&mut self, states: Vec) -> StreamExecutorResult<()> + pub async fn set_states(&mut self, states: Vec) -> StreamExecutorResult<()> where SS: SplitMetaData, { - if states.is_empty() { - // TODO should be a clear Error Code - bail!("states require not null"); - } else { - for split_impl in states { - self.set(split_impl.id(), split_impl.encode_to_json()) - .await?; - } + for split_impl in states { + self.set(split_impl.id(), split_impl.encode_to_json()) + .await?; } Ok(()) } - pub async fn trim_state(&mut self, to_trim: &[SS]) -> StreamExecutorResult<()> - where - SS: SplitMetaData, - { + pub async fn trim_state(&mut self, to_trim: &[SplitImpl]) -> StreamExecutorResult<()> { for split in to_trim { tracing::info!("trimming source state for split {}", split.id()); self.delete(split.id()).await?; @@ -228,8 +210,9 @@ impl SourceStateTableHandler { } } -// align with schema defined in `LogicalSource::infer_internal_table_catalog`. The function is used -// for test purpose and should not be used in production. +/// align with schema defined in `LogicalSource::infer_internal_table_catalog`. The function is used +/// for test purpose and should not be used in production. +#[cfg(test)] pub fn default_source_internal_table(id: u32) -> PbTable { let make_column = |column_type: TypeName, column_id: i32| -> ColumnCatalog { ColumnCatalog { @@ -325,7 +308,7 @@ pub(crate) mod tests { state_table_handler.init_epoch(epoch_1); state_table_handler - .take_snapshot(vec![split_impl.clone()]) + .set_states(vec![split_impl.clone()]) .await?; state_table_handler.state_store.commit(epoch_2).await?; diff --git a/src/stream/src/executor/stateless_simple_agg.rs b/src/stream/src/executor/stateless_simple_agg.rs index 23c3b999978ef..c6bca07df8cad 100644 --- a/src/stream/src/executor/stateless_simple_agg.rs +++ b/src/stream/src/executor/stateless_simple_agg.rs @@ -16,7 +16,6 @@ use futures::StreamExt; use futures_async_stream::try_stream; use itertools::Itertools; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::aggregate::{ build_retractable, AggCall, AggregateState, BoxedAggregateFunction, @@ -29,28 +28,16 @@ use crate::error::StreamResult; pub struct StatelessSimpleAggExecutor { _ctx: ActorContextRef, - pub(super) info: ExecutorInfo, - pub(super) input: Box, + pub(super) input: Executor, + pub(super) schema: Schema, pub(super) aggs: Vec, pub(super) agg_calls: Vec, } -impl Executor for StatelessSimpleAggExecutor { +impl Execute for StatelessSimpleAggExecutor { fn execute(self: Box) -> BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } impl StatelessSimpleAggExecutor { @@ -73,7 +60,7 @@ impl StatelessSimpleAggExecutor { let StatelessSimpleAggExecutor { _ctx, input, - info, + schema, aggs, agg_calls, } = self; @@ -94,7 +81,7 @@ impl StatelessSimpleAggExecutor { if is_dirty { is_dirty = false; - let mut builders = info.schema.create_array_builders(1); + let mut builders = schema.create_array_builders(1); for ((agg, state), builder) in aggs .iter() .zip_eq_fast(states.iter_mut()) @@ -124,15 +111,15 @@ impl StatelessSimpleAggExecutor { impl StatelessSimpleAggExecutor { pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: Box, + input: Executor, + schema: Schema, agg_calls: Vec, ) -> StreamResult { let aggs = agg_calls.iter().map(build_retractable).try_collect()?; Ok(StatelessSimpleAggExecutor { _ctx: ctx, - info, input, + schema, aggs, agg_calls, }) @@ -151,34 +138,24 @@ mod tests { use super::*; use crate::executor::test_utils::agg_executor::generate_agg_schema; use crate::executor::test_utils::MockSource; - use crate::executor::{Executor, StatelessSimpleAggExecutor}; + use crate::executor::{Execute, StatelessSimpleAggExecutor}; #[tokio::test] async fn test_no_chunk() { let schema = schema_test_utils::ii(); - let (mut tx, source) = MockSource::channel(schema, vec![2]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![2]); tx.push_barrier(test_epoch(1), false); tx.push_barrier(test_epoch(2), false); tx.push_barrier(test_epoch(3), false); let agg_calls = vec![AggCall::from_pretty("(count:int8)")]; let schema = generate_agg_schema(&source, &agg_calls, None); - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "StatelessSimpleAggExecutor".to_string(), - }; - let simple_agg = Box::new( - StatelessSimpleAggExecutor::new( - ActorContext::create(123), - info, - Box::new(source), - agg_calls, - ) - .unwrap(), - ); - let mut simple_agg = simple_agg.execute(); + let simple_agg = + StatelessSimpleAggExecutor::new(ActorContext::for_test(123), source, schema, agg_calls) + .unwrap(); + let mut simple_agg = simple_agg.boxed().execute(); assert_matches!( simple_agg.next().await.unwrap().unwrap(), @@ -197,7 +174,8 @@ mod tests { #[tokio::test] async fn test_local_simple_agg() { let schema = schema_test_utils::iii(); - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk\ + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![2]); tx.push_barrier(test_epoch(1), false); tx.push_chunk(StreamChunk::from_pretty( " I I I @@ -221,22 +199,11 @@ mod tests { AggCall::from_pretty("(sum:int8 $1:int8)"), ]; let schema = generate_agg_schema(&source, &agg_calls, None); - let info = ExecutorInfo { - schema, - pk_indices: vec![], - identity: "StatelessSimpleAggExecutor".to_string(), - }; - let simple_agg = Box::new( - StatelessSimpleAggExecutor::new( - ActorContext::create(123), - info, - Box::new(source), - agg_calls, - ) - .unwrap(), - ); - let mut simple_agg = simple_agg.execute(); + let simple_agg = + StatelessSimpleAggExecutor::new(ActorContext::for_test(123), source, schema, agg_calls) + .unwrap(); + let mut simple_agg = simple_agg.boxed().execute(); // Consume the init barrier simple_agg.next().await.unwrap().unwrap(); diff --git a/src/stream/src/executor/stream_reader.rs b/src/stream/src/executor/stream_reader.rs index 8d401b3649352..6bbf76d0bf85c 100644 --- a/src/stream/src/executor/stream_reader.rs +++ b/src/stream/src/executor/stream_reader.rs @@ -18,10 +18,8 @@ use std::task::Poll; use either::Either; use futures::stream::{select_with_strategy, BoxStream, PollNext, SelectWithStrategy}; use futures::{Stream, StreamExt, TryStreamExt}; -use futures_async_stream::try_stream; -use risingwave_connector::source::BoxTryStream; -use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; +use crate::executor::error::StreamExecutorResult; use crate::executor::Message; type ExecutorMessageStream = BoxStream<'static, StreamExecutorResult>; @@ -48,26 +46,14 @@ pub(super) struct StreamReaderWithPause { } impl StreamReaderWithPause { - /// Receive messages from the reader. Hang up on error. - #[try_stream(ok = M, error = StreamExecutorError)] - async fn data_stream(stream: BoxTryStream) { - // TODO: support stack trace for Stream - #[for_await] - for m in stream { - match m { - Ok(m) => yield m, - Err(err) => { - return Err(StreamExecutorError::connector_error(err)); - } - } - } - } - /// Construct a `StreamReaderWithPause` with one stream receiving barrier messages (and maybe /// other types of messages) and the other receiving data only (no barrier). - pub fn new(message_stream: ExecutorMessageStream, data_stream: BoxTryStream) -> Self { + pub fn new( + message_stream: ExecutorMessageStream, + data_stream: impl Stream> + Send + 'static, + ) -> Self { let message_stream_arm = message_stream.map_ok(Either::Left).boxed(); - let data_stream_arm = Self::data_stream(data_stream).map_ok(Either::Right).boxed(); + let data_stream_arm = data_stream.map_ok(Either::Right).boxed(); let inner = Self::new_inner(message_stream_arm, data_stream_arm); Self { inner, @@ -89,7 +75,10 @@ impl StreamReaderWithPause { } /// Replace the data stream with a new one for given `stream`. Used for split change. - pub fn replace_data_stream(&mut self, data_stream: BoxTryStream) { + pub fn replace_data_stream( + &mut self, + data_stream: impl Stream> + Send + 'static, + ) { // Take the barrier receiver arm. let barrier_receiver_arm = std::mem::replace( self.inner.get_mut().0, @@ -100,7 +89,7 @@ impl StreamReaderWithPause { // to ensure the internal state of the `SelectWithStrategy` is reset. (#6300) self.inner = Self::new_inner( barrier_receiver_arm, - Self::data_stream(data_stream).map_ok(Either::Right).boxed(), + data_stream.map_ok(Either::Right).boxed(), ); } @@ -144,12 +133,11 @@ mod tests { use risingwave_common::array::StreamChunk; use risingwave_common::transaction::transaction_id::TxnId; use risingwave_common::util::epoch::test_epoch; - use risingwave_connector::source::StreamChunkWithState; - use risingwave_source::TableDmlHandle; + use risingwave_dml::TableDmlHandle; use tokio::sync::mpsc; use super::*; - use crate::executor::{barrier_to_message_stream, Barrier}; + use crate::executor::{barrier_to_message_stream, Barrier, StreamExecutorError}; const TEST_TRANSACTION_ID1: TxnId = 0; const TEST_TRANSACTION_ID2: TxnId = 1; @@ -162,7 +150,10 @@ mod tests { let table_dml_handle = TableDmlHandle::new(vec![], TEST_DML_CHANNEL_INIT_PERMITS); - let source_stream = table_dml_handle.stream_reader().into_data_stream_for_test(); + let source_stream = table_dml_handle + .stream_reader() + .into_data_stream_for_test() + .map_err(StreamExecutorError::from); let mut write_handle1 = table_dml_handle .write_handle(TEST_SESSION_ID, TEST_TRANSACTION_ID1) @@ -172,8 +163,7 @@ mod tests { .unwrap(); let barrier_stream = barrier_to_message_stream(barrier_rx).boxed(); - let stream = - StreamReaderWithPause::::new(barrier_stream, source_stream); + let stream = StreamReaderWithPause::::new(barrier_stream, source_stream); pin_mut!(stream); macro_rules! next { @@ -193,7 +183,7 @@ mod tests { .write_chunk(StreamChunk::default()) .await .unwrap(); - // We don't call end() here, since we test `StreamChunkWithState` instead of `TxnMsg`. + // We don't call end() here, since we test `StreamChunk` instead of `TxnMsg`. assert_matches!(next!().unwrap(), Either::Right(_)); // Write a barrier, and we should receive it. @@ -215,7 +205,7 @@ mod tests { .write_chunk(StreamChunk::default()) .await .unwrap(); - // We don't call end() here, since we test `StreamChunkWithState` instead of `TxnMsg`. + // We don't call end() here, since we test `StreamChunk` instead of `TxnMsg`. // We should receive the barrier. assert_matches!(next!().unwrap(), Either::Left(_)); diff --git a/src/stream/src/executor/subscription.rs b/src/stream/src/executor/subscription.rs new file mode 100644 index 0000000000000..1dd47f89c43d9 --- /dev/null +++ b/src/stream/src/executor/subscription.rs @@ -0,0 +1,132 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use core::str::FromStr; +use core::time::Duration; +use std::collections::HashMap; + +use futures::prelude::stream::StreamExt; +use futures_async_stream::try_stream; +use risingwave_common::types::{Interval, Timestamptz}; +use risingwave_common::util::epoch::Epoch; +use risingwave_storage::store::LocalStateStore; +use tokio::time::Instant; + +use super::{ + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + StreamExecutorError, StreamExecutorResult, +}; +use crate::common::log_store_impl::kv_log_store::ReaderTruncationOffsetType; +use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter; + +const EXECUTE_GC_INTERVAL: u64 = 3600; + +pub struct SubscriptionExecutor { + actor_context: ActorContextRef, + input: Executor, + log_store: SubscriptionLogStoreWriter, + retention_seconds: i64, +} + +impl SubscriptionExecutor { + #[allow(clippy::too_many_arguments)] + #[expect(clippy::unused_async)] + pub async fn new( + actor_context: ActorContextRef, + input: Executor, + log_store: SubscriptionLogStoreWriter, + properties: HashMap, + ) -> StreamExecutorResult { + let retention_seconds_str = properties.get("retention").ok_or_else(|| { + StreamExecutorError::serde_error("Subscription retention time not set.".to_string()) + })?; + let retention_seconds = (Interval::from_str(retention_seconds_str) + .map_err(|_| { + StreamExecutorError::serde_error( + "Retention needs to be set in Interval format".to_string(), + ) + })? + .epoch_in_micros() + / 1000000) as i64; + + Ok(Self { + actor_context, + input, + log_store, + retention_seconds, + }) + } + + #[try_stream(ok = Message, error = StreamExecutorError)] + async fn execute_inner(mut self) { + let mut input = self.input.execute(); + + let barrier = expect_first_barrier(&mut input).await?; + self.log_store.init(barrier.epoch, false).await?; + + // The first barrier message should be propagated. + yield Message::Barrier(barrier); + + let mut next_truncate_time = Instant::now() + Duration::from_secs(EXECUTE_GC_INTERVAL); + + #[for_await] + for msg in input { + let msg = msg?; + yield match msg { + Message::Watermark(w) => Message::Watermark(w), + Message::Chunk(chunk) => { + if chunk.cardinality() == 0 { + // empty chunk + continue; + } + self.log_store.write_chunk(chunk.clone())?; + Message::Chunk(chunk) + } + Message::Barrier(barrier) => { + let truncate_offset: Option = if next_truncate_time + < Instant::now() + { + let truncate_timestamptz = Timestamptz::from_secs(barrier.get_curr_epoch().as_timestamptz().timestamp() - self.retention_seconds).ok_or_else(||{StreamExecutorError::from("Subscription retention time calculation error: timestamp is out of range.".to_string())})?; + let epoch = + Epoch::from_unix_millis(truncate_timestamptz.timestamp_millis() as u64); + next_truncate_time = + Instant::now() + Duration::from_secs(EXECUTE_GC_INTERVAL); + Some((epoch.0, None)) + } else { + None + }; + self.log_store + .flush_current_epoch( + barrier.epoch.curr, + barrier.kind.is_checkpoint(), + truncate_offset, + ) + .await?; + + if let Some(vnode_bitmap) = + barrier.as_update_vnode_bitmap(self.actor_context.id) + { + self.log_store.update_vnode_bitmap(vnode_bitmap)?; + } + Message::Barrier(barrier) + } + } + } + } +} +impl Execute for SubscriptionExecutor { + fn execute(self: Box) -> BoxedMessageStream { + self.execute_inner().boxed() + } +} diff --git a/src/stream/src/executor/subtask.rs b/src/stream/src/executor/subtask.rs index 0130a73fc411a..fea9644b151f3 100644 --- a/src/stream/src/executor/subtask.rs +++ b/src/stream/src/executor/subtask.rs @@ -20,7 +20,7 @@ use tokio::sync::mpsc::error::SendError; use tokio_stream::wrappers::ReceiverStream; use super::actor::spawn_blocking_drop_stream; -use super::{BoxedExecutor, Executor, ExecutorInfo, Message, MessageStreamItem}; +use super::{Execute, Executor, Message, MessageStreamItem}; use crate::task::ActorId; /// Handle used to drive the subtask. @@ -29,31 +29,13 @@ pub type SubtaskHandle = impl Future + Send + 'static; /// The thin wrapper for subtask-wrapped executor, containing a channel to receive the messages from /// the subtask. pub struct SubtaskRxExecutor { - info: ExecutorInfo, - rx: mpsc::Receiver, } -impl Executor for SubtaskRxExecutor { +impl Execute for SubtaskRxExecutor { fn execute(self: Box) -> super::BoxedMessageStream { ReceiverStream::new(self.rx).boxed() } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> ExecutorInfo { - self.info.clone() - } } /// Wrap an executor into a subtask and a thin receiver executor, connected by a channel with a @@ -62,15 +44,9 @@ impl Executor for SubtaskRxExecutor { /// Used when there're multiple stateful executors in an actor. These subtasks can be concurrently /// executed to improve the I/O performance, while the computing resource can be still bounded to a /// single thread. -pub fn wrap(input: BoxedExecutor, actor_id: ActorId) -> (SubtaskHandle, SubtaskRxExecutor) { +pub fn wrap(input: Executor, actor_id: ActorId) -> (SubtaskHandle, SubtaskRxExecutor) { let (tx, rx) = mpsc::channel(1); - let rx_executor = SubtaskRxExecutor { - info: ExecutorInfo { - identity: "SubtaskRxExecutor".to_owned(), - ..input.info() - }, - rx, - }; + let rx_executor = SubtaskRxExecutor { rx }; let handle = async move { let mut input = input.execute(); diff --git a/src/stream/src/executor/temporal_join.rs b/src/stream/src/executor/temporal_join.rs index 099abe658f615..b4147ec5b1cbd 100644 --- a/src/stream/src/executor/temporal_join.rs +++ b/src/stream/src/executor/temporal_join.rs @@ -15,18 +15,17 @@ use std::alloc::Global; use std::collections::hash_map::Entry; use std::collections::HashMap; -use std::ops::{Deref, DerefMut}; use std::pin::pin; use std::sync::Arc; use either::Either; use futures::stream::{self, PollNext}; use futures::{pin_mut, StreamExt, TryStreamExt}; -use futures_async_stream::try_stream; +use futures_async_stream::{for_await, try_stream}; use local_stats_alloc::{SharedStatsAlloc, StatsAlloc}; use lru::DefaultHasher; use risingwave_common::array::{Op, StreamChunk}; -use risingwave_common::catalog::Schema; +use risingwave_common::buffer::BitmapBuilder; use risingwave_common::estimate_size::{EstimateSize, KvSize}; use risingwave_common::hash::{HashKey, NullBitmap}; use risingwave_common::row::{OwnedRow, Row, RowExt}; @@ -39,22 +38,24 @@ use risingwave_storage::table::batch_table::storage_table::StorageTable; use risingwave_storage::table::TableIter; use risingwave_storage::StateStore; +use super::join::{JoinType, JoinTypePrimitive}; use super::{ - Barrier, Executor, ExecutorInfo, Message, MessageStream, StreamExecutorError, + Barrier, Execute, ExecutorInfo, Message, MessageStream, StreamExecutorError, StreamExecutorResult, }; use crate::cache::{cache_may_stale, new_with_hasher_in, ManagedLruCache}; use crate::common::metrics::MetricsInfo; -use crate::common::JoinStreamChunkBuilder; +use crate::executor::join::builder::JoinStreamChunkBuilder; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{ActorContextRef, BoxedExecutor, JoinType, JoinTypePrimitive, Watermark}; +use crate::executor::{ActorContextRef, Executor, Watermark}; use crate::task::AtomicU64Ref; pub struct TemporalJoinExecutor { ctx: ActorContextRef, + #[allow(dead_code)] info: ExecutorInfo, - left: BoxedExecutor, - right: BoxedExecutor, + left: Executor, + right: Executor, right_table: TemporalSide, left_join_keys: Vec, right_join_keys: Vec, @@ -107,99 +108,84 @@ impl JoinEntry { } } -struct JoinEntryWrapper(Option); - -impl EstimateSize for JoinEntryWrapper { - fn estimated_heap_size(&self) -> usize { - self.0.estimated_heap_size() - } -} - -impl JoinEntryWrapper { - const MESSAGE: &'static str = "the state should always be `Some`"; - - /// Take the value out of the wrapper. Panic if the value is `None`. - pub fn take(&mut self) -> JoinEntry { - self.0.take().expect(Self::MESSAGE) - } -} - -impl Deref for JoinEntryWrapper { - type Target = JoinEntry; - - fn deref(&self) -> &Self::Target { - self.0.as_ref().expect(Self::MESSAGE) - } -} - -impl DerefMut for JoinEntryWrapper { - fn deref_mut(&mut self) -> &mut Self::Target { - self.0.as_mut().expect(Self::MESSAGE) - } -} - struct TemporalSide { source: StorageTable, table_stream_key_indices: Vec, table_output_indices: Vec, - cache: ManagedLruCache>, + cache: ManagedLruCache>, ctx: ActorContextRef, join_key_data_types: Vec, } impl TemporalSide { - /// Lookup the temporal side table and return a `JoinEntry` which could be empty if there are no - /// matched records. - async fn lookup(&mut self, key: &K, epoch: HummockEpoch) -> StreamExecutorResult { + /// Fetch records from temporal side table and ensure the entry in the cache. + /// If already exists, the entry will be promoted. + async fn fetch_or_promote_keys( + &mut self, + keys: impl Iterator, + epoch: HummockEpoch, + ) -> StreamExecutorResult<()> { let table_id_str = self.source.table_id().to_string(); let actor_id_str = self.ctx.id.to_string(); let fragment_id_str = self.ctx.id.to_string(); - self.ctx - .streaming_metrics - .temporal_join_total_query_cache_count - .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) - .inc(); - - let res = if self.cache.contains(key) { - let mut state = self.cache.peek_mut(key).unwrap(); - state.take() - } else { - // cache miss + + let mut futs = Vec::with_capacity(keys.size_hint().1.unwrap_or(0)); + for key in keys { self.ctx .streaming_metrics - .temporal_join_cache_miss_count + .temporal_join_total_query_cache_count .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) .inc(); - let pk_prefix = key.deserialize(&self.join_key_data_types)?; + if self.cache.get(key).is_none() { + self.ctx + .streaming_metrics + .temporal_join_cache_miss_count + .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) + .inc(); + + futs.push(async { + let pk_prefix = key.deserialize(&self.join_key_data_types)?; + + let iter = self + .source + .batch_iter_with_pk_bounds( + HummockReadEpoch::NoWait(epoch), + &pk_prefix, + .., + false, + PrefetchOptions::default(), + ) + .await?; - let iter = self - .source - .batch_iter_with_pk_bounds( - HummockReadEpoch::NoWait(epoch), - &pk_prefix, - .., - false, - PrefetchOptions::default(), - ) - .await?; - - let mut entry = JoinEntry::default(); - - pin_mut!(iter); - while let Some(row) = iter.next_row().await? { - entry.insert( - row.as_ref() - .project(&self.table_stream_key_indices) - .into_owned_row(), - row.project(&self.table_output_indices).into_owned_row(), - ); + let mut entry = JoinEntry::default(); + + pin_mut!(iter); + while let Some(row) = iter.next_row().await? { + entry.insert( + row.as_ref() + .project(&self.table_stream_key_indices) + .into_owned_row(), + row.project(&self.table_output_indices).into_owned_row(), + ); + } + let key = key.clone(); + Ok((key, entry)) as StreamExecutorResult<_> + }); } + } - entry - }; + #[for_await] + for res in stream::iter(futs).buffered(16) { + let (key, entry) = res?; + self.cache.put(key, entry); + } - Ok(res) + Ok(()) + } + + fn force_peek(&self, key: &K) -> &JoinEntry { + self.cache.peek(key).expect("key should exists") } fn update( @@ -209,7 +195,7 @@ impl TemporalSide { right_stream_key_indices: &[usize], ) -> StreamExecutorResult<()> { for chunk in chunks { - let keys = K::build(join_keys, chunk.data_chunk())?; + let keys = K::build_many(join_keys, chunk.data_chunk()); for (r, key) in chunk.rows_with_holes().zip_eq_debug(keys.into_iter()) { let Some((op, row)) = r else { continue; @@ -229,10 +215,6 @@ impl TemporalSide { } Ok(()) } - - pub fn insert_back(&mut self, key: K, state: JoinEntry) { - self.cache.put(key, JoinEntryWrapper(Some(state))); - } } enum InternalMessage { @@ -280,7 +262,7 @@ async fn internal_messages_until_barrier(stream: impl MessageStream, expected_ba // any number of `InternalMessage::Chunk(left_chunk)` and followed by // `InternalMessage::Barrier(right_chunks, barrier)`. #[try_stream(ok = InternalMessage, error = StreamExecutorError)] -async fn align_input(left: Box, right: Box) { +async fn align_input(left: Executor, right: Executor) { let mut left = pin!(left.execute()); let mut right = pin!(right.execute()); // Keep producing intervals until stream exhaustion or errors. @@ -327,13 +309,177 @@ async fn align_input(left: Box, right: Box) { } } +mod phase1 { + use futures_async_stream::try_stream; + use risingwave_common::array::stream_chunk_builder::StreamChunkBuilder; + use risingwave_common::array::{Op, StreamChunk}; + use risingwave_common::hash::{HashKey, NullBitmap}; + use risingwave_common::row::{self, Row, RowExt}; + use risingwave_common::types::{DataType, DatumRef}; + use risingwave_common::util::iter_util::ZipEqDebug; + use risingwave_hummock_sdk::HummockEpoch; + use risingwave_storage::StateStore; + + use super::{StreamExecutorError, TemporalSide}; + + pub(super) trait Phase1Evaluation { + /// Called when a matched row is found. + #[must_use = "consume chunk if produced"] + fn append_matched_row( + op: Op, + builder: &mut StreamChunkBuilder, + left_row: impl Row, + right_row: impl Row, + ) -> Option; + + /// Called when all matched rows of a join key are appended. + #[must_use = "consume chunk if produced"] + fn match_end( + builder: &mut StreamChunkBuilder, + op: Op, + left_row: impl Row, + right_size: usize, + matched: bool, + ) -> Option; + } + + pub(super) struct Inner; + pub(super) struct LeftOuter; + pub(super) struct LeftOuterWithCond; + + impl Phase1Evaluation for Inner { + fn append_matched_row( + op: Op, + builder: &mut StreamChunkBuilder, + left_row: impl Row, + right_row: impl Row, + ) -> Option { + builder.append_row(op, left_row.chain(right_row)) + } + + fn match_end( + _builder: &mut StreamChunkBuilder, + _op: Op, + _left_row: impl Row, + _right_size: usize, + _matched: bool, + ) -> Option { + None + } + } + + impl Phase1Evaluation for LeftOuter { + fn append_matched_row( + op: Op, + builder: &mut StreamChunkBuilder, + left_row: impl Row, + right_row: impl Row, + ) -> Option { + builder.append_row(op, left_row.chain(right_row)) + } + + fn match_end( + builder: &mut StreamChunkBuilder, + op: Op, + left_row: impl Row, + right_size: usize, + matched: bool, + ) -> Option { + if !matched { + // If no rows matched, a marker row should be inserted. + builder.append_row( + op, + left_row.chain(row::repeat_n(DatumRef::None, right_size)), + ) + } else { + None + } + } + } + + impl Phase1Evaluation for LeftOuterWithCond { + fn append_matched_row( + op: Op, + builder: &mut StreamChunkBuilder, + left_row: impl Row, + right_row: impl Row, + ) -> Option { + builder.append_row(op, left_row.chain(right_row)) + } + + fn match_end( + builder: &mut StreamChunkBuilder, + op: Op, + left_row: impl Row, + right_size: usize, + _matched: bool, + ) -> Option { + // A marker row should always be inserted and mark as invisible for non-lookup filters evaluation. + // The row will be converted to visible in the further steps if no rows matched after all filters evaluated. + builder.append_row_invisible( + op, + left_row.chain(row::repeat_n(DatumRef::None, right_size)), + ) + } + } + + #[try_stream(ok = StreamChunk, error = StreamExecutorError)] + #[allow(clippy::too_many_arguments)] + pub(super) async fn handle_chunk<'a, K: HashKey, S: StateStore, E: Phase1Evaluation>( + chunk_size: usize, + right_size: usize, + full_schema: Vec, + epoch: HummockEpoch, + left_join_keys: &'a [usize], + right_table: &'a mut TemporalSide, + null_matched: &'a K::Bitmap, + chunk: StreamChunk, + ) { + let mut builder = StreamChunkBuilder::new(chunk_size, full_schema); + let keys = K::build_many(left_join_keys, chunk.data_chunk()); + let to_fetch_keys = chunk + .visibility() + .iter() + .zip_eq_debug(keys.iter()) + .filter_map(|(vis, key)| if vis { Some(key) } else { None }); + right_table + .fetch_or_promote_keys(to_fetch_keys, epoch) + .await?; + for (r, key) in chunk.rows_with_holes().zip_eq_debug(keys.into_iter()) { + let Some((op, left_row)) = r else { + continue; + }; + let mut matched = false; + if key.null_bitmap().is_subset(null_matched) + && let join_entry = right_table.force_peek(&key) + && !join_entry.is_empty() + { + matched = true; + for right_row in join_entry.cached.values() { + if let Some(chunk) = + E::append_matched_row(op, &mut builder, left_row, right_row) + { + yield chunk; + } + } + } + if let Some(chunk) = E::match_end(&mut builder, op, left_row, right_size, matched) { + yield chunk; + } + } + if let Some(chunk) = builder.take() { + yield chunk; + } + } +} + impl TemporalJoinExecutor { #[allow(clippy::too_many_arguments)] pub fn new( ctx: ActorContextRef, info: ExecutorInfo, - left: BoxedExecutor, - right: BoxedExecutor, + left: Executor, + right: Executor, table: StorageTable, left_join_keys: Vec, right_join_keys: Vec, @@ -386,12 +532,25 @@ impl TemporalJoinExecutor } } + fn apply_indices_map(chunk: StreamChunk, indices: &[usize]) -> StreamChunk { + let (data_chunk, ops) = chunk.into_parts(); + let (columns, vis) = data_chunk.into_parts(); + let output_columns = indices + .iter() + .cloned() + .map(|idx| columns[idx].clone()) + .collect(); + StreamChunk::with_visibility(ops, output_columns, vis) + } + #[try_stream(ok = Message, error = StreamExecutorError)] async fn into_stream(mut self) { - let (left_map, right_map) = JoinStreamChunkBuilder::get_i2o_mapping( + let right_size = self.right.schema().len(); + + let (left_map, _right_map) = JoinStreamChunkBuilder::get_i2o_mapping( &self.output_indices, self.left.schema().len(), - self.right.schema().len(), + right_size, ); let left_to_output: HashMap = HashMap::from_iter(left_map.iter().cloned()); @@ -405,6 +564,14 @@ impl TemporalJoinExecutor let table_id_str = self.right_table.source.table_id().to_string(); let actor_id_str = self.ctx.id.to_string(); let fragment_id_str = self.ctx.fragment_id.to_string(); + let full_schema: Vec<_> = self + .left + .schema() + .data_types() + .into_iter() + .chain(self.right.schema().data_types().into_iter()) + .collect(); + #[for_await] for msg in align_input(self.left, self.right) { self.right_table.cache.evict(); @@ -419,51 +586,106 @@ impl TemporalJoinExecutor yield Message::Watermark(watermark.with_idx(output_watermark_col_idx)); } InternalMessage::Chunk(chunk) => { - let mut builder = JoinStreamChunkBuilder::new( - self.chunk_size, - self.info.schema.data_types(), - left_map.clone(), - right_map.clone(), - ); let epoch = prev_epoch.expect("Chunk data should come after some barrier."); - let keys = K::build(&self.left_join_keys, chunk.data_chunk())?; - for (r, key) in chunk.rows_with_holes().zip_eq_debug(keys.into_iter()) { - let Some((op, left_row)) = r else { - continue; - }; - if key.null_bitmap().is_subset(&null_matched) - && let join_entry = self.right_table.lookup(&key, epoch).await? - && !join_entry.is_empty() - { - for right_row in join_entry.cached.values() { - // check join condition - let ok = if let Some(ref mut cond) = self.condition { - let concat_row = left_row.chain(&right_row).into_owned_row(); - cond.eval_row_infallible(&concat_row) - .await - .map(|s| *s.as_bool()) - .unwrap_or(false) - } else { - true - }; - if ok { - if let Some(chunk) = builder.append_row(op, left_row, right_row) - { - yield Message::Chunk(chunk); + let full_schema = full_schema.clone(); + + if T == JoinType::Inner { + let st1 = phase1::handle_chunk::( + self.chunk_size, + right_size, + full_schema, + epoch, + &self.left_join_keys, + &mut self.right_table, + &null_matched, + chunk, + ); + #[for_await] + for chunk in st1 { + let chunk = chunk?; + let new_chunk = if let Some(ref cond) = self.condition { + let (data_chunk, ops) = chunk.into_parts(); + let passed_bitmap = cond.eval_infallible(&data_chunk).await; + let passed_bitmap = + Arc::unwrap_or_clone(passed_bitmap).into_bool().to_bitmap(); + let (columns, vis) = data_chunk.into_parts(); + let new_vis = vis & passed_bitmap; + StreamChunk::with_visibility(ops, columns, new_vis) + } else { + chunk + }; + let new_chunk = + Self::apply_indices_map(new_chunk, &self.output_indices); + yield Message::Chunk(new_chunk); + } + } else if let Some(ref cond) = self.condition { + // Joined result without evaluating non-lookup conditions. + let st1 = phase1::handle_chunk::( + self.chunk_size, + right_size, + full_schema, + epoch, + &self.left_join_keys, + &mut self.right_table, + &null_matched, + chunk, + ); + let mut matched_count = 0usize; + #[for_await] + for chunk in st1 { + let chunk = chunk?; + let (data_chunk, ops) = chunk.into_parts(); + let passed_bitmap = cond.eval_infallible(&data_chunk).await; + let passed_bitmap = + Arc::unwrap_or_clone(passed_bitmap).into_bool().to_bitmap(); + let (columns, vis) = data_chunk.into_parts(); + let mut new_vis = BitmapBuilder::with_capacity(vis.len()); + for (passed, not_match_end) in + passed_bitmap.iter().zip_eq_debug(vis.iter()) + { + let is_match_end = !not_match_end; + let vis = if is_match_end && matched_count == 0 { + // Nothing is matched, so the marker row should be visible. + true + } else if is_match_end { + // reset the count + matched_count = 0; + // rows found, so the marker row should be invisible. + false + } else { + if passed { + matched_count += 1; } - } - } - // Insert back the state taken from ht. - self.right_table.insert_back(key.clone(), join_entry); - } else if T == JoinType::LeftOuter { - if let Some(chunk) = builder.append_row_update(op, left_row) { - yield Message::Chunk(chunk); + passed + }; + new_vis.append(vis); } + let new_chunk = Self::apply_indices_map( + StreamChunk::with_visibility(ops, columns, new_vis.finish()), + &self.output_indices, + ); + yield Message::Chunk(new_chunk); + } + // The last row should always be marker row, + assert_eq!(matched_count, 0); + } else { + let st1 = phase1::handle_chunk::( + self.chunk_size, + right_size, + full_schema, + epoch, + &self.left_join_keys, + &mut self.right_table, + &null_matched, + chunk, + ); + #[for_await] + for chunk in st1 { + let chunk = chunk?; + let new_chunk = Self::apply_indices_map(chunk, &self.output_indices); + yield Message::Chunk(new_chunk); } - } - if let Some(chunk) = builder.take() { - yield Message::Chunk(chunk); } } InternalMessage::Barrier(updates, barrier) => { @@ -488,22 +710,10 @@ impl TemporalJoinExecutor } } -impl Executor +impl Execute for TemporalJoinExecutor { fn execute(self: Box) -> super::BoxedMessageStream { self.into_stream().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } diff --git a/src/stream/src/executor/test_utils.rs b/src/stream/src/executor/test_utils.rs index 4b1b9bf944370..785384307ecf4 100644 --- a/src/stream/src/executor/test_utils.rs +++ b/src/stream/src/executor/test_utils.rs @@ -22,8 +22,8 @@ use tokio::sync::mpsc; use super::error::StreamExecutorError; use super::{ - Barrier, BoxedMessageStream, Executor, Message, MessageStream, PkIndices, StreamChunk, - StreamExecutorResult, Watermark, + Barrier, BoxedMessageStream, Execute, Executor, ExecutorInfo, Message, MessageStream, + StreamChunk, StreamExecutorResult, Watermark, }; pub mod prelude { @@ -41,12 +41,10 @@ pub mod prelude { pub use crate::common::table::state_table::StateTable; pub use crate::executor::test_utils::expr::build_from_pretty; pub use crate::executor::test_utils::{MessageSender, MockSource, StreamExecutorTestExt}; - pub use crate::executor::{ActorContext, BoxedMessageStream, Executor, PkIndices}; + pub use crate::executor::{ActorContext, BoxedMessageStream, Execute, PkIndices}; } pub struct MockSource { - schema: Schema, - pk_indices: PkIndices, rx: mpsc::UnboundedReceiver, /// Whether to send a `Stop` barrier on stream finish. @@ -108,20 +106,15 @@ impl MessageSender { impl std::fmt::Debug for MockSource { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("MockSource") - .field("schema", &self.schema) - .field("pk_indices", &self.pk_indices) - .finish() + f.debug_struct("MockSource").finish() } } impl MockSource { #[allow(dead_code)] - pub fn channel(schema: Schema, pk_indices: PkIndices) -> (MessageSender, Self) { + pub fn channel() -> (MessageSender, Self) { let (tx, rx) = mpsc::unbounded_channel(); let source = Self { - schema, - pk_indices, rx, stop_on_finish: true, }; @@ -129,16 +122,16 @@ impl MockSource { } #[allow(dead_code)] - pub fn with_messages(schema: Schema, pk_indices: PkIndices, msgs: Vec) -> Self { - let (tx, source) = Self::channel(schema, pk_indices); + pub fn with_messages(msgs: Vec) -> Self { + let (tx, source) = Self::channel(); for msg in msgs { tx.0.send(msg).unwrap(); } source } - pub fn with_chunks(schema: Schema, pk_indices: PkIndices, chunks: Vec) -> Self { - let (tx, source) = Self::channel(schema, pk_indices); + pub fn with_chunks(chunks: Vec) -> Self { + let (tx, source) = Self::channel(); for chunk in chunks { tx.0.send(Message::Chunk(chunk)).unwrap(); } @@ -154,6 +147,17 @@ impl MockSource { } } + pub fn into_executor(self, schema: Schema, pk_indices: Vec) -> Executor { + Executor::new( + ExecutorInfo { + schema, + pk_indices, + identity: "MockSource".to_string(), + }, + self.boxed(), + ) + } + #[try_stream(ok = Message, error = StreamExecutorError)] async fn execute_inner(mut self: Box) { let mut epoch = test_epoch(1); @@ -169,22 +173,10 @@ impl MockSource { } } -impl Executor for MockSource { +impl Execute for MockSource { fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &Schema { - &self.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.pk_indices - } - - fn identity(&self) -> &str { - "MockSource" - } } /// `row_nonnull` builds a `OwnedRow` with concrete values. @@ -296,14 +288,14 @@ pub mod agg_executor { }; use crate::executor::aggregation::AggStateStorage; use crate::executor::{ - ActorContext, ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, HashAggExecutor, - PkIndices, SimpleAggExecutor, + ActorContext, ActorContextRef, Executor, ExecutorInfo, HashAggExecutor, PkIndices, + SimpleAggExecutor, }; - /// Generate agg executor's schema from `input`, `agg_calls` and `group_key_indices`. + /// Generate aggExecuter's schema from `input`, `agg_calls` and `group_key_indices`. /// For [`crate::executor::HashAggExecutor`], the group key indices should be provided. pub fn generate_agg_schema( - input: &dyn Executor, + input_ref: &Executor, agg_calls: &[AggCall], group_key_indices: Option<&[usize]>, ) -> Schema { @@ -314,7 +306,7 @@ pub mod agg_executor { let fields = if let Some(key_indices) = group_key_indices { let keys = key_indices .iter() - .map(|idx| input.schema().fields[*idx].clone()); + .map(|idx| input_ref.schema().fields[*idx].clone()); keys.chain(aggs).collect() } else { @@ -332,7 +324,7 @@ pub mod agg_executor { agg_call: &AggCall, group_key_indices: &[usize], pk_indices: &[usize], - input_ref: &dyn Executor, + input_ref: &Executor, is_append_only: bool, ) -> AggStateStorage { match agg_call.kind { @@ -403,7 +395,7 @@ pub mod agg_executor { table_id: TableId, agg_calls: &[AggCall], group_key_indices: &[usize], - input_ref: &dyn Executor, + input_ref: &Executor, ) -> StateTable { let input_fields = input_ref.schema().fields(); @@ -442,7 +434,7 @@ pub mod agg_executor { #[allow(clippy::too_many_arguments)] pub async fn new_boxed_hash_agg_executor( store: S, - input: Box, + input: Executor, is_append_only: bool, agg_calls: Vec, row_count_index: usize, @@ -451,7 +443,7 @@ pub mod agg_executor { extreme_cache_size: usize, emit_on_window_close: bool, executor_id: u64, - ) -> Box { + ) -> Executor { let mut storages = Vec::with_capacity(agg_calls.iter().len()); for (idx, agg_call) in agg_calls.iter().enumerate() { storages.push( @@ -461,7 +453,7 @@ pub mod agg_executor { agg_call, &group_key_indices, &pk_indices, - input.as_ref(), + &input, is_append_only, ) .await, @@ -473,23 +465,23 @@ pub mod agg_executor { TableId::new(agg_calls.len() as u32), &agg_calls, &group_key_indices, - input.as_ref(), + &input, ) .await; - let schema = generate_agg_schema(input.as_ref(), &agg_calls, Some(&group_key_indices)); + let schema = generate_agg_schema(&input, &agg_calls, Some(&group_key_indices)); let info = ExecutorInfo { schema, pk_indices, identity: format!("HashAggExecutor {:X}", executor_id), }; - HashAggExecutor::::new(AggExecutorArgs { + let exec = HashAggExecutor::::new(AggExecutorArgs { version: PbAggNodeVersion::Max, input, - actor_ctx: ActorContext::create(123), - info, + actor_ctx: ActorContext::for_test(123), + info: info.clone(), extreme_cache_size, @@ -507,21 +499,21 @@ pub mod agg_executor { emit_on_window_close, }, }) - .unwrap() - .boxed() + .unwrap(); + (info, exec).into() } #[allow(clippy::too_many_arguments)] pub async fn new_boxed_simple_agg_executor( actor_ctx: ActorContextRef, store: S, - input: BoxedExecutor, + input: Executor, is_append_only: bool, agg_calls: Vec, row_count_index: usize, pk_indices: PkIndices, executor_id: u64, - ) -> Box { + ) -> Executor { let storages = future::join_all(agg_calls.iter().enumerate().map(|(idx, agg_call)| { create_agg_state_storage( store.clone(), @@ -529,7 +521,7 @@ pub mod agg_executor { agg_call, &[], &pk_indices, - input.as_ref(), + &input, is_append_only, ) })) @@ -540,23 +532,23 @@ pub mod agg_executor { TableId::new(agg_calls.len() as u32), &agg_calls, &[], - input.as_ref(), + &input, ) .await; - let schema = generate_agg_schema(input.as_ref(), &agg_calls, None); + let schema = generate_agg_schema(&input, &agg_calls, None); let info = ExecutorInfo { schema, pk_indices, identity: format!("SimpleAggExecutor {:X}", executor_id), }; - SimpleAggExecutor::new(AggExecutorArgs { + let exec = SimpleAggExecutor::new(AggExecutorArgs { version: PbAggNodeVersion::Max, input, actor_ctx, - info, + info: info.clone(), extreme_cache_size: 1024, @@ -568,8 +560,8 @@ pub mod agg_executor { watermark_epoch: Arc::new(AtomicU64::new(0)), extra: SimpleAggExecutorExtraArgs {}, }) - .unwrap() - .boxed() + .unwrap(); + (info, exec).into() } } diff --git a/src/stream/src/executor/top_n/group_top_n.rs b/src/stream/src/executor/top_n/group_top_n.rs index 74728fd86a54d..0720559dc54d0 100644 --- a/src/stream/src/executor/top_n/group_top_n.rs +++ b/src/stream/src/executor/top_n/group_top_n.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::Bitmap; +use risingwave_common::catalog::Schema; use risingwave_common::hash::HashKey; use risingwave_common::row::RowExt; use risingwave_common::util::epoch::EpochPair; @@ -32,7 +33,7 @@ use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; use crate::task::AtomicU64Ref; pub type GroupTopNExecutor = @@ -41,9 +42,9 @@ pub type GroupTopNExecutor = impl GroupTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -55,7 +56,7 @@ impl GroupTopNExecutor GroupTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// `LIMIT XXX`. None means no limit. limit: usize, @@ -97,7 +98,7 @@ pub struct InnerGroupTopNExecutor InnerGroupTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -113,12 +114,11 @@ impl InnerGroupTopNExecutor::new(state_table, cache_key_serde.clone()); Ok(Self { - info, + schema, offset: offset_and_limit.0, limit: offset_and_limit.1, managed_state, @@ -164,7 +164,7 @@ where async fn apply_chunk(&mut self, chunk: StreamChunk) -> StreamExecutorResult { let mut res_ops = Vec::with_capacity(self.limit); let mut res_rows = Vec::with_capacity(self.limit); - let keys = K::build(&self.group_by, chunk.data_chunk())?; + let keys = K::build_many(&self.group_by, chunk.data_chunk()); let table_id_str = self.managed_state.table().table_id().to_string(); let actor_id_str = self.ctx.id.to_string(); let fragment_id_str = self.ctx.fragment_id.to_string(); @@ -191,7 +191,7 @@ where .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) .inc(); let mut topn_cache = - TopNCache::new(self.offset, self.limit, self.info().schema.data_types()); + TopNCache::new(self.offset, self.limit, self.schema.data_types()); self.managed_state .init_topn_cache(Some(group_key), &mut topn_cache) .await?; @@ -227,7 +227,7 @@ where .group_top_n_cached_entry_count .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) .set(self.caches.len() as i64); - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -238,10 +238,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn update_epoch(&mut self, epoch: u64) { self.caches.update_epoch(epoch); } @@ -290,7 +286,7 @@ mod tests { use super::*; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Barrier, Message}; + use crate::executor::{ActorContext, Barrier, Execute, Message}; fn create_schema() -> Schema { Schema { @@ -356,24 +352,21 @@ mod tests { vec![chunk0, chunk1, chunk2, chunk3] } - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = create_stream_chunks(); let schema = create_schema(); - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(5))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(5))), + ]) + .into_executor(schema, pk_indices()) } #[tokio::test] @@ -389,26 +382,20 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), // this includes group key as prefix - identity: "GroupTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - GroupTopNExecutor::::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (0, 2), - order_by_1(), - vec![1], - state_table, - Arc::new(AtomicU64::new(0)), - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = GroupTopNExecutor::::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 2), + order_by_1(), + vec![1], + state_table, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -491,26 +478,20 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), // this includes group key as prefix - identity: "GroupTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - GroupTopNExecutor::::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (1, 2), - order_by_1(), - vec![1], - state_table, - Arc::new(AtomicU64::new(0)), - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = GroupTopNExecutor::::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 2), + order_by_1(), + vec![1], + state_table, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -586,26 +567,20 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), // this includes group key as prefix - identity: "GroupTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - GroupTopNExecutor::::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (0, 2), - order_by_2(), - vec![1, 2], - state_table, - Arc::new(AtomicU64::new(0)), - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = GroupTopNExecutor::::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 2), + order_by_2(), + vec![1, 2], + state_table, + Arc::new(AtomicU64::new(0)), + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); diff --git a/src/stream/src/executor/top_n/group_top_n_appendonly.rs b/src/stream/src/executor/top_n/group_top_n_appendonly.rs index 44522ddfb31ff..b50897f874bcf 100644 --- a/src/stream/src/executor/top_n/group_top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/group_top_n_appendonly.rs @@ -16,6 +16,7 @@ use std::sync::Arc; use risingwave_common::array::{Op, StreamChunk}; use risingwave_common::buffer::Bitmap; +use risingwave_common::catalog::Schema; use risingwave_common::hash::HashKey; use risingwave_common::row::{RowDeserializer, RowExt}; use risingwave_common::util::epoch::EpochPair; @@ -31,7 +32,7 @@ use crate::common::metrics::MetricsInfo; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; use crate::task::AtomicU64Ref; /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need @@ -45,9 +46,9 @@ impl { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -59,7 +60,7 @@ impl input, ctx: ctx.clone(), inner: InnerAppendOnlyGroupTopNExecutor::new( - info, + schema, storage_key, offset_and_limit, order_by, @@ -73,7 +74,7 @@ impl } pub struct InnerAppendOnlyGroupTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// `LIMIT XXX`. None means no limit. limit: usize, @@ -103,7 +104,7 @@ impl { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -119,12 +120,11 @@ impl "GroupTopN", ); - let cache_key_serde = - create_cache_key_serde(&storage_key, &info.schema, &order_by, &group_by); + let cache_key_serde = create_cache_key_serde(&storage_key, &schema, &order_by, &group_by); let managed_state = ManagedTopNState::::new(state_table, cache_key_serde.clone()); Ok(Self { - info, + schema, offset: offset_and_limit.0, limit: offset_and_limit.1, managed_state, @@ -145,9 +145,9 @@ where async fn apply_chunk(&mut self, chunk: StreamChunk) -> StreamExecutorResult { let mut res_ops = Vec::with_capacity(self.limit); let mut res_rows = Vec::with_capacity(self.limit); - let keys = K::build(&self.group_by, chunk.data_chunk())?; + let keys = K::build_many(&self.group_by, chunk.data_chunk()); - let data_types = self.info().schema.data_types(); + let data_types = self.schema.data_types(); let row_deserializer = RowDeserializer::new(data_types.clone()); let table_id_str = self.managed_state.table().table_id().to_string(); let actor_id_str = self.ctx.id.to_string(); @@ -197,7 +197,7 @@ where .group_top_n_appendonly_cached_entry_count .with_label_values(&[&table_id_str, &actor_id_str, &fragment_id_str]) .set(self.caches.len() as i64); - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -208,10 +208,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - fn update_vnode_bitmap(&mut self, vnode_bitmap: Arc) { let cache_may_stale = self.managed_state.update_vnode_bitmap(vnode_bitmap); if cache_may_stale { diff --git a/src/stream/src/executor/top_n/top_n_appendonly.rs b/src/stream/src/executor/top_n/top_n_appendonly.rs index a7c854177c8f7..229c87422a0bb 100644 --- a/src/stream/src/executor/top_n/top_n_appendonly.rs +++ b/src/stream/src/executor/top_n/top_n_appendonly.rs @@ -13,6 +13,7 @@ // limitations under the License. use risingwave_common::array::{Op, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::row::{RowDeserializer, RowExt}; use risingwave_common::util::epoch::EpochPair; use risingwave_common::util::sort_util::ColumnOrder; @@ -24,7 +25,7 @@ use super::{ManagedTopNState, TopNCache, NO_GROUP_KEY}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; /// If the input is append-only, `AppendOnlyGroupTopNExecutor` does not need /// to keep all the rows seen. As long as a record @@ -38,9 +39,9 @@ pub type AppendOnlyTopNExecutor = impl AppendOnlyTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -50,7 +51,7 @@ impl AppendOnlyTopNExecutor input, ctx, inner: InnerAppendOnlyTopNExecutor::new( - info, + schema, storage_key, offset_and_limit, order_by, @@ -61,7 +62,7 @@ impl AppendOnlyTopNExecutor } pub struct InnerAppendOnlyTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// The storage key indices of the `TopNExecutor` storage_key_indices: PkIndices, @@ -80,7 +81,7 @@ pub struct InnerAppendOnlyTopNExecutor { impl InnerAppendOnlyTopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -89,12 +90,12 @@ impl InnerAppendOnlyTopNExecutor::new(state_table, cache_key_serde.clone()); - let data_types = info.schema.data_types(); + let data_types = schema.data_types(); Ok(Self { - info, + schema, managed_state, storage_key_indices: storage_key.into_iter().map(|op| op.column_index).collect(), cache: TopNCache::new(num_offset, num_limit, data_types), @@ -111,7 +112,7 @@ where async fn apply_chunk(&mut self, chunk: StreamChunk) -> StreamExecutorResult { let mut res_ops = Vec::with_capacity(self.cache.limit); let mut res_rows = Vec::with_capacity(self.cache.limit); - let data_types = self.info().schema.data_types(); + let data_types = self.schema.data_types(); let row_deserializer = RowDeserializer::new(data_types); // apply the chunk to state table for (op, row_ref) in chunk.rows() { @@ -128,7 +129,7 @@ where )?; } - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -139,10 +140,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - async fn init(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { self.managed_state.init_epoch(epoch); self.managed_state @@ -171,7 +168,7 @@ mod tests { use super::AppendOnlyTopNExecutor; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table; use crate::executor::test_utils::MockSource; - use crate::executor::{ActorContext, Barrier, Executor, ExecutorInfo, Message, PkIndices}; + use crate::executor::{ActorContext, Barrier, Execute, Executor, Message, PkIndices}; fn create_stream_chunks() -> Vec { let chunk1 = StreamChunk::from_pretty( @@ -224,21 +221,17 @@ mod tests { vec![0, 1] } - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = create_stream_chunks(); - let schema = create_schema(); - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(std::mem::take(&mut chunks[2])), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(std::mem::take(&mut chunks[2])), + ]) + .into_executor(create_schema(), pk_indices()) } #[tokio::test] @@ -252,24 +245,18 @@ mod tests { ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "AppendOnlyTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - AppendOnlyTopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key, - (0, 5), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = AppendOnlyTopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key, + (0, 5), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init epoch top_n_executor.next().await.unwrap().unwrap(); @@ -339,24 +326,18 @@ mod tests { ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "AppendOnlyTopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - AppendOnlyTopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (3, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = AppendOnlyTopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (3, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init epoch top_n_executor.next().await.unwrap().unwrap(); diff --git a/src/stream/src/executor/top_n/top_n_plain.rs b/src/stream/src/executor/top_n/top_n_plain.rs index 01705df660665..f9f183164093a 100644 --- a/src/stream/src/executor/top_n/top_n_plain.rs +++ b/src/stream/src/executor/top_n/top_n_plain.rs @@ -13,6 +13,7 @@ // limitations under the License. use risingwave_common::array::{Op, StreamChunk}; +use risingwave_common::catalog::Schema; use risingwave_common::row::RowExt; use risingwave_common::util::epoch::EpochPair; use risingwave_common::util::sort_util::ColumnOrder; @@ -23,7 +24,7 @@ use super::{ManagedTopNState, TopNCache, TopNCacheTrait}; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; use crate::executor::error::StreamExecutorResult; -use crate::executor::{ActorContextRef, Executor, ExecutorInfo, PkIndices, Watermark}; +use crate::executor::{ActorContextRef, Executor, PkIndices, Watermark}; /// `TopNExecutor` works with input with modification, it keeps all the data /// records/rows that have been seen, and returns topN records overall. @@ -33,9 +34,9 @@ pub type TopNExecutor = impl TopNExecutor { #[allow(clippy::too_many_arguments)] pub fn new( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -45,7 +46,7 @@ impl TopNExecutor { input, ctx, inner: InnerTopNExecutor::new( - info, + schema, storage_key, offset_and_limit, order_by, @@ -61,16 +62,16 @@ impl TopNExecutor { #[allow(clippy::too_many_arguments)] #[cfg(test)] pub fn new_with_ties_for_test( - input: Box, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, state_table: StateTable, ) -> StreamResult { let mut inner = - InnerTopNExecutor::new(info, storage_key, offset_and_limit, order_by, state_table)?; + InnerTopNExecutor::new(schema, storage_key, offset_and_limit, order_by, state_table)?; inner.cache.high_capacity = 2; @@ -79,7 +80,7 @@ impl TopNExecutor { } pub struct InnerTopNExecutor { - info: ExecutorInfo, + schema: Schema, /// The storage key indices of the `TopNExecutor` storage_key_indices: PkIndices, @@ -103,7 +104,7 @@ impl InnerTopNExecutor { /// into `CacheKey`. #[allow(clippy::too_many_arguments)] pub fn new( - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -112,12 +113,12 @@ impl InnerTopNExecutor { let num_offset = offset_and_limit.0; let num_limit = offset_and_limit.1; - let cache_key_serde = create_cache_key_serde(&storage_key, &info.schema, &order_by, &[]); + let cache_key_serde = create_cache_key_serde(&storage_key, &schema, &order_by, &[]); let managed_state = ManagedTopNState::::new(state_table, cache_key_serde.clone()); - let data_types = info.schema.data_types(); + let data_types = schema.data_types(); Ok(Self { - info, + schema, managed_state, storage_key_indices: storage_key.into_iter().map(|op| op.column_index).collect(), cache: TopNCache::new(num_offset, num_limit, data_types), @@ -162,7 +163,7 @@ where } } } - generate_output(res_rows, res_ops, &self.info().schema) + generate_output(res_rows, res_ops, &self.schema) } async fn flush_data(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { @@ -173,10 +174,6 @@ where self.managed_state.try_flush().await } - fn info(&self) -> &ExecutorInfo { - &self.info - } - async fn init(&mut self, epoch: EpochPair) -> StreamExecutorResult<()> { self.managed_state.init_epoch(epoch); self.managed_state @@ -209,7 +206,7 @@ mod tests { use risingwave_common::util::epoch::test_epoch; use super::*; - use crate::executor::ActorContext; + use crate::executor::{ActorContext, Execute}; fn create_stream_chunks() -> Vec { let chunk1 = StreamChunk::from_pretty( " I I @@ -268,24 +265,21 @@ mod tests { vec![0, 1] } - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = create_stream_chunks(); let schema = create_schema(); - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(5))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(5))), + ]) + .into_executor(schema, pk_indices()) } #[tokio::test] @@ -298,24 +292,18 @@ mod tests { ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (3, 1000), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (3, 1000), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -399,24 +387,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (0, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -512,24 +494,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, true>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (0, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, true>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -624,24 +600,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (3, 4), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (3, 4), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -720,9 +690,9 @@ mod tests { use super::*; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table_from_state_store; - use crate::executor::ActorContext; + use crate::executor::{ActorContext, Execute}; - fn create_source_new() -> Box { + fn create_source_new() -> Executor { let mut chunks = vec![ StreamChunk::from_pretty( " I I I I @@ -751,22 +721,19 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + ]) + .into_executor(schema, pk_indices()) } - fn create_source_new_before_recovery() -> Box { - let mut chunks = vec![ + fn create_source_new_before_recovery() -> Executor { + let mut chunks = [ StreamChunk::from_pretty( " I I I I + 1 1 4 1001", @@ -784,20 +751,17 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + ]) + .into_executor(schema, pk_indices()) } - fn create_source_new_after_recovery() -> Box { - let mut chunks = vec![ + fn create_source_new_after_recovery() -> Executor { + let mut chunks = [ StreamChunk::from_pretty( " I I I I + 1 9 1 1003 @@ -817,16 +781,13 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + ]) + .into_executor(schema, pk_indices()) } fn storage_key() -> Vec { @@ -858,24 +819,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (1, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -942,24 +897,18 @@ mod tests { ) .await; let source = create_source_new_before_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (1, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -1002,24 +951,18 @@ mod tests { // recovery let source = create_source_new_after_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor_after_recovery = Box::new( - TopNExecutor::<_, false>::new( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (1, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor_after_recovery.execute(); + let schema = source.schema().clone(); + let top_n_executor_after_recovery = TopNExecutor::<_, false>::new( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (1, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor_after_recovery.boxed().execute(); // barrier assert_matches!( @@ -1064,9 +1007,9 @@ mod tests { use super::*; use crate::executor::test_utils::top_n_executor::create_in_memory_state_table_from_state_store; - use crate::executor::ActorContext; + use crate::executor::{ActorContext, Execute}; - fn create_source() -> Box { + fn create_source() -> Executor { let mut chunks = vec![ StreamChunk::from_pretty( " I I @@ -1101,18 +1044,15 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Chunk(std::mem::take(&mut chunks[2])), - Message::Chunk(std::mem::take(&mut chunks[3])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Chunk(std::mem::take(&mut chunks[2])), + Message::Chunk(std::mem::take(&mut chunks[3])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + ]) + .into_executor(schema, pk_indices()) } fn storage_key() -> Vec { @@ -1138,24 +1078,18 @@ mod tests { &pk_indices(), ) .await; - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::new_with_ties_for_test( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (0, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::new_with_ties_for_test( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -1215,8 +1149,8 @@ mod tests { ); } - fn create_source_before_recovery() -> Box { - let mut chunks = vec![ + fn create_source_before_recovery() -> Executor { + let mut chunks = [ StreamChunk::from_pretty( " I I + 1 0 @@ -1242,20 +1176,17 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(1))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(2))), + ]) + .into_executor(schema, pk_indices()) } - fn create_source_after_recovery() -> Box { - let mut chunks = vec![ + fn create_source_after_recovery() -> Executor { + let mut chunks = [ StreamChunk::from_pretty( " I I - 1 0", @@ -1271,16 +1202,13 @@ mod tests { Field::unnamed(DataType::Int64), ], }; - Box::new(MockSource::with_messages( - schema, - pk_indices(), - vec![ - Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), - Message::Chunk(std::mem::take(&mut chunks[0])), - Message::Chunk(std::mem::take(&mut chunks[1])), - Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), - ], - )) + MockSource::with_messages(vec![ + Message::Barrier(Barrier::new_test_barrier(test_epoch(3))), + Message::Chunk(std::mem::take(&mut chunks[0])), + Message::Chunk(std::mem::take(&mut chunks[1])), + Message::Barrier(Barrier::new_test_barrier(test_epoch(4))), + ]) + .into_executor(schema, pk_indices()) } #[tokio::test] @@ -1294,24 +1222,18 @@ mod tests { ) .await; let source = create_source_before_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor = Box::new( - TopNExecutor::new_with_ties_for_test( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (0, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor.execute(); + let schema = source.schema().clone(); + let top_n_executor = TopNExecutor::new_with_ties_for_test( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor.boxed().execute(); // consume the init barrier top_n_executor.next().await.unwrap().unwrap(); @@ -1357,24 +1279,18 @@ mod tests { // recovery let source = create_source_after_recovery(); - let info = ExecutorInfo { - schema: source.schema().clone(), - pk_indices: source.pk_indices().to_vec(), - identity: "TopNExecutor 1".to_string(), - }; - let top_n_executor_after_recovery = Box::new( - TopNExecutor::new_with_ties_for_test( - source as Box, - ActorContext::create(0), - info, - storage_key(), - (0, 3), - order_by(), - state_table, - ) - .unwrap(), - ); - let mut top_n_executor = top_n_executor_after_recovery.execute(); + let schema = source.schema().clone(); + let top_n_executor_after_recovery = TopNExecutor::new_with_ties_for_test( + source, + ActorContext::for_test(0), + schema, + storage_key(), + (0, 3), + order_by(), + state_table, + ) + .unwrap(); + let mut top_n_executor = top_n_executor_after_recovery.boxed().execute(); // barrier assert_matches!( diff --git a/src/stream/src/executor/top_n/top_n_state.rs b/src/stream/src/executor/top_n/top_n_state.rs index 6ab2dbd1e1740..58bdd867b684e 100644 --- a/src/stream/src/executor/top_n/top_n_state.rs +++ b/src/stream/src/executor/top_n/top_n_state.rs @@ -365,8 +365,8 @@ mod tests { let row2_bytes = serialize_pk_to_cache_key(row2.clone(), &cache_key_serde); let row3_bytes = serialize_pk_to_cache_key(row3.clone(), &cache_key_serde); let row4_bytes = serialize_pk_to_cache_key(row4.clone(), &cache_key_serde); - let rows = vec![row1, row2, row3, row4]; - let ordered_rows = vec![row1_bytes, row2_bytes, row3_bytes, row4_bytes]; + let rows = [row1, row2, row3, row4]; + let ordered_rows = [row1_bytes, row2_bytes, row3_bytes, row4_bytes]; managed_state.insert(rows[3].clone()); // now ("ab", 4) @@ -447,7 +447,7 @@ mod tests { let row3_bytes = serialize_pk_to_cache_key(row3.clone(), &cache_key_serde); let row4_bytes = serialize_pk_to_cache_key(row4.clone(), &cache_key_serde); let row5_bytes = serialize_pk_to_cache_key(row5.clone(), &cache_key_serde); - let rows = vec![row1, row2, row3, row4, row5]; + let rows = [row1, row2, row3, row4, row5]; let ordered_rows = vec![row1_bytes, row2_bytes, row3_bytes, row4_bytes, row5_bytes]; let mut cache = TopNCache::::new(1, 1, data_types); diff --git a/src/stream/src/executor/top_n/utils.rs b/src/stream/src/executor/top_n/utils.rs index b2310f0d352b1..bbd956cde2168 100644 --- a/src/stream/src/executor/top_n/utils.rs +++ b/src/stream/src/executor/top_n/utils.rs @@ -30,8 +30,8 @@ use risingwave_common::util::sort_util::ColumnOrder; use super::CacheKey; use crate::executor::error::{StreamExecutorError, StreamExecutorResult}; use crate::executor::{ - expect_first_barrier, ActorContextRef, BoxedExecutor, BoxedMessageStream, Executor, - ExecutorInfo, Message, PkIndicesRef, Watermark, + expect_first_barrier, ActorContextRef, BoxedMessageStream, Execute, Executor, Message, + Watermark, }; pub trait TopNExecutorBase: Send + 'static { @@ -50,8 +50,6 @@ pub trait TopNExecutorBase: Send + 'static { /// Flush the buffered chunk to the storage backend. fn try_flush_data(&mut self) -> impl Future> + Send; - fn info(&self) -> &ExecutorInfo; - /// Update the vnode bitmap for the state table and manipulate the cache if necessary, only used /// by Group Top-N since it's distributed. fn update_vnode_bitmap(&mut self, _vnode_bitmap: Arc) { @@ -72,34 +70,18 @@ pub trait TopNExecutorBase: Send + 'static { /// The struct wraps a [`TopNExecutorBase`] pub struct TopNExecutorWrapper { - pub(super) input: BoxedExecutor, + pub(super) input: Executor, pub(super) ctx: ActorContextRef, pub(super) inner: E, } -impl Executor for TopNExecutorWrapper +impl Execute for TopNExecutorWrapper where E: TopNExecutorBase, { fn execute(self: Box) -> BoxedMessageStream { self.top_n_executor_execute().boxed() } - - fn schema(&self) -> &Schema { - &self.inner.info().schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.inner.info().pk_indices - } - - fn identity(&self) -> &str { - &self.inner.info().identity - } - - fn info(&self) -> ExecutorInfo { - self.inner.info().clone() - } } impl TopNExecutorWrapper diff --git a/src/stream/src/executor/union.rs b/src/stream/src/executor/union.rs index deb2a4cd3b9e8..5881474d90197 100644 --- a/src/stream/src/executor/union.rs +++ b/src/stream/src/executor/union.rs @@ -17,57 +17,37 @@ use std::pin::Pin; use std::task::{Context, Poll}; use futures::stream::{FusedStream, FuturesUnordered}; -use futures::StreamExt; +use futures::{Stream, StreamExt}; use futures_async_stream::try_stream; use pin_project::pin_project; -use risingwave_common::catalog::Schema; use super::watermark::BufferedWatermarks; -use super::*; -use crate::executor::{BoxedMessageStream, ExecutorInfo}; +use super::{ + Barrier, BoxedMessageStream, Execute, Executor, Message, MessageStreamItem, StreamExecutorError, +}; /// `UnionExecutor` merges data from multiple inputs. pub struct UnionExecutor { - info: ExecutorInfo, - inputs: Vec, + inputs: Vec, } impl std::fmt::Debug for UnionExecutor { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("UnionExecutor") - .field("schema", &self.info.schema) - .field("pk_indices", &self.info.pk_indices) - .finish() + f.debug_struct("UnionExecutor").finish() } } impl UnionExecutor { - pub fn new(info: ExecutorInfo, inputs: Vec) -> Self { - Self { info, inputs } + pub fn new(inputs: Vec) -> Self { + Self { inputs } } } -impl Executor for UnionExecutor { +impl Execute for UnionExecutor { fn execute(self: Box) -> BoxedMessageStream { let streams = self.inputs.into_iter().map(|e| e.execute()).collect(); merge(streams).boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> ExecutorInfo { - self.info.clone() - } } #[pin_project] @@ -170,9 +150,11 @@ mod tests { use async_stream::try_stream; use risingwave_common::array::stream_chunk::StreamChunkTestExt; use risingwave_common::array::StreamChunk; + use risingwave_common::types::{DataType, ScalarImpl}; use risingwave_common::util::epoch::test_epoch; use super::*; + use crate::executor::Watermark; #[tokio::test] async fn union() { diff --git a/src/stream/src/executor/utils.rs b/src/stream/src/executor/utils.rs index 8ff9f8f29576d..18b64633a3e12 100644 --- a/src/stream/src/executor/utils.rs +++ b/src/stream/src/executor/utils.rs @@ -13,40 +13,19 @@ // limitations under the License. use futures::StreamExt; -use risingwave_common::catalog::Schema; use risingwave_common::metrics::LabelGuardedIntCounter; use crate::executor::monitor::StreamingMetrics; -use crate::executor::{BoxedMessageStream, Executor, ExecutorInfo, PkIndicesRef}; +use crate::executor::{BoxedMessageStream, Execute}; use crate::task::{ActorId, FragmentId}; #[derive(Default)] -pub struct DummyExecutor { - pub info: ExecutorInfo, -} +pub struct DummyExecutor; -impl DummyExecutor { - pub fn new(info: ExecutorInfo) -> Self { - Self { info } - } -} - -impl Executor for DummyExecutor { +impl Execute for DummyExecutor { fn execute(self: Box) -> BoxedMessageStream { futures::stream::pending().boxed() } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } } pub(crate) struct ActorInputMetrics { diff --git a/src/stream/src/executor/values.rs b/src/stream/src/executor/values.rs index 9eda43e72a41e..79fdf890cc7e7 100644 --- a/src/stream/src/executor/values.rs +++ b/src/stream/src/executor/values.rs @@ -24,10 +24,7 @@ use risingwave_common::util::iter_util::ZipEqFast; use risingwave_expr::expr::NonStrictExpression; use tokio::sync::mpsc::UnboundedReceiver; -use super::{ - ActorContextRef, Barrier, BoxedMessageStream, Executor, ExecutorInfo, Message, PkIndicesRef, - StreamExecutorError, -}; +use super::{ActorContextRef, Barrier, BoxedMessageStream, Execute, Message, StreamExecutorError}; use crate::task::CreateMviewProgress; const DEFAULT_CHUNK_SIZE: usize = 1024; @@ -36,8 +33,8 @@ const DEFAULT_CHUNK_SIZE: usize = 1024; /// May refractor with `BarrierRecvExecutor` in the near future. pub struct ValuesExecutor { ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, // Receiver of barrier channel. barrier_receiver: UnboundedReceiver, progress: CreateMviewProgress, @@ -49,14 +46,14 @@ impl ValuesExecutor { /// Currently hard-code the `pk_indices` as the last column. pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, progress: CreateMviewProgress, rows: Vec>, barrier_receiver: UnboundedReceiver, ) -> Self { Self { ctx, - info, + schema, progress, barrier_receiver, rows: rows.into_iter(), @@ -64,9 +61,9 @@ impl ValuesExecutor { } #[try_stream(ok = Message, error = StreamExecutorError)] - async fn into_stream(self) { + async fn execute_inner(self) { let Self { - info, + schema, mut progress, mut barrier_receiver, mut rows, @@ -96,7 +93,7 @@ impl ValuesExecutor { } } - let cardinality = info.schema.len(); + let cardinality = schema.len(); ensure!(cardinality > 0); while !rows.is_empty() { // We need a one row chunk rather than an empty chunk because constant @@ -105,7 +102,7 @@ impl ValuesExecutor { let one_row_chunk = DataChunk::new_dummy(1); let chunk_size = DEFAULT_CHUNK_SIZE.min(rows.len()); - let mut array_builders = info.schema.create_array_builders(chunk_size); + let mut array_builders = schema.create_array_builders(chunk_size); for row in rows.by_ref().take(chunk_size) { for (expr, builder) in row.into_iter().zip_eq_fast(&mut array_builders) { let out = expr.eval_infallible(&one_row_chunk).await; @@ -135,21 +132,9 @@ impl ValuesExecutor { } } -impl Executor for ValuesExecutor { +impl Execute for ValuesExecutor { fn execute(self: Box) -> BoxedMessageStream { - self.into_stream().boxed() - } - - fn schema(&self) -> &Schema { - &self.info.schema - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity + self.execute_inner().boxed() } } @@ -168,7 +153,7 @@ mod tests { use super::ValuesExecutor; use crate::executor::test_utils::StreamExecutorTestExt; - use crate::executor::{ActorContext, AddMutation, Barrier, Executor, ExecutorInfo, Mutation}; + use crate::executor::{ActorContext, AddMutation, Barrier, Execute, Mutation}; use crate::task::{CreateMviewProgress, LocalBarrierManager}; #[tokio::test] @@ -207,15 +192,9 @@ mod tests { .iter() // for each column .map(|col| Field::unnamed(col.return_type())) .collect::(); - let pk_indices = vec![schema.len() - 1]; - let info = ExecutorInfo { - schema, - pk_indices, - identity: "ValuesExecutor".to_string(), - }; let values_executor_struct = ValuesExecutor::new( - ActorContext::create(actor_id), - info, + ActorContext::for_test(actor_id), + schema, progress, vec![exprs .into_iter() diff --git a/src/stream/src/executor/watermark_filter.rs b/src/stream/src/executor/watermark_filter.rs index 0f418955cf1a1..e53545dd394f6 100644 --- a/src/stream/src/executor/watermark_filter.rs +++ b/src/stream/src/executor/watermark_filter.rs @@ -36,9 +36,7 @@ use risingwave_storage::StateStore; use super::error::StreamExecutorError; use super::filter::FilterExecutor; -use super::{ - ActorContextRef, BoxedExecutor, Executor, ExecutorInfo, Message, StreamExecutorResult, -}; +use super::{ActorContextRef, Execute, Executor, ExecutorInfo, Message, StreamExecutorResult}; use crate::common::table::state_table::StateTable; use crate::executor::{expect_first_barrier, Watermark}; use crate::task::ActorEvalErrorReport; @@ -48,59 +46,49 @@ use crate::task::ActorEvalErrorReport; /// filtered. pub struct WatermarkFilterExecutor { ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + input: Executor, /// The expression used to calculate the watermark value. watermark_expr: NonStrictExpression, /// The column we should generate watermark and filter on. event_time_col_idx: usize, table: StateTable, global_watermark_table: StorageTable, + + eval_error_report: ActorEvalErrorReport, } impl WatermarkFilterExecutor { pub fn new( ctx: ActorContextRef, - info: ExecutorInfo, - input: BoxedExecutor, + info: &ExecutorInfo, + input: Executor, watermark_expr: NonStrictExpression, event_time_col_idx: usize, table: StateTable, global_watermark_table: StorageTable, ) -> Self { + let eval_error_report = ActorEvalErrorReport { + actor_context: ctx.clone(), + identity: Arc::from(info.identity.as_ref()), + }; + Self { ctx, - info, input, watermark_expr, event_time_col_idx, table, global_watermark_table, + eval_error_report, } } } -impl Executor for WatermarkFilterExecutor { +impl Execute for WatermarkFilterExecutor { fn execute(self: Box) -> super::BoxedMessageStream { self.execute_inner().boxed() } - - fn schema(&self) -> &risingwave_common::catalog::Schema { - &self.info.schema - } - - fn pk_indices(&self) -> super::PkIndicesRef<'_> { - &self.info.pk_indices - } - - fn identity(&self) -> &str { - &self.info.identity - } - - fn info(&self) -> ExecutorInfo { - self.info.clone() - } } impl WatermarkFilterExecutor { @@ -111,16 +99,11 @@ impl WatermarkFilterExecutor { event_time_col_idx, watermark_expr, ctx, - info, mut table, mut global_watermark_table, + eval_error_report, } = *self; - let eval_error_report = ActorEvalErrorReport { - actor_context: ctx.clone(), - identity: info.identity.into(), - }; - let watermark_type = watermark_expr.return_type(); assert_eq!( watermark_type, @@ -263,17 +246,16 @@ impl WatermarkFilterExecutor { // Persist the watermark when checkpoint arrives. if let Some(watermark) = current_watermark.clone() { for vnode in table.vnodes().clone().iter_vnodes() { - let pk = Some(ScalarImpl::Int16(vnode.to_scalar())); + let pk = vnode.to_datum(); let row = [pk, Some(watermark.clone())]; // This is an upsert. table.insert(row); } } - table.commit(barrier.epoch).await?; - } else { - table.commit_no_data_expected(barrier.epoch); } + table.commit(barrier.epoch).await?; + if barrier.kind.is_checkpoint() { if idle_input { // Align watermark @@ -458,7 +440,7 @@ mod tests { async fn create_watermark_filter_executor( mem_state: MemoryStateStore, - ) -> (BoxedExecutor, MessageSender) { + ) -> (Box, MessageSender) { let schema = Schema { fields: vec![ Field::unnamed(DataType::Int16), // pk @@ -478,7 +460,8 @@ mod tests { ) .await; - let (tx, source) = MockSource::channel(schema, vec![0]); + let (tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![0]); let info = ExecutorInfo { schema: source.schema().clone(), @@ -488,9 +471,9 @@ mod tests { ( WatermarkFilterExecutor::new( - ActorContext::create(123), - info, - source.boxed(), + ActorContext::for_test(123), + &info, + source, watermark_expr, 1, table, diff --git a/src/stream/src/executor/wrapper.rs b/src/stream/src/executor/wrapper.rs index ebd3f6d9bfa73..dddd94da5ab73 100644 --- a/src/stream/src/executor/wrapper.rs +++ b/src/stream/src/executor/wrapper.rs @@ -15,9 +15,8 @@ use std::sync::Arc; use futures::StreamExt; -use risingwave_common::catalog::Schema; -use super::*; +use super::{ActorContextRef, BoxedMessageStream, Execute, Executor, ExecutorInfo, MessageStream}; mod epoch_check; mod epoch_provide; @@ -27,7 +26,7 @@ mod update_check; /// [`WrapperExecutor`] will do some sanity checks and logging for the wrapped executor. pub struct WrapperExecutor { - input: BoxedExecutor, + input: Executor, actor_ctx: ActorContextRef, @@ -36,7 +35,7 @@ pub struct WrapperExecutor { impl WrapperExecutor { pub fn new( - input: BoxedExecutor, + input: Executor, actor_ctx: ActorContextRef, enable_executor_row_count: bool, ) -> Self { @@ -88,9 +87,9 @@ impl WrapperExecutor { } } -impl Executor for WrapperExecutor { +impl Execute for WrapperExecutor { fn execute(self: Box) -> BoxedMessageStream { - let info = Arc::new(self.input.info()); + let info = Arc::new(self.input.info().clone()); Self::wrap( self.enable_executor_row_count, info, @@ -101,7 +100,7 @@ impl Executor for WrapperExecutor { } fn execute_with_epoch(self: Box, epoch: u64) -> BoxedMessageStream { - let info = Arc::new(self.input.info()); + let info = Arc::new(self.input.info().clone()); Self::wrap( self.enable_executor_row_count, info, @@ -110,16 +109,4 @@ impl Executor for WrapperExecutor { ) .boxed() } - - fn schema(&self) -> &Schema { - self.input.schema() - } - - fn pk_indices(&self) -> PkIndicesRef<'_> { - self.input.pk_indices() - } - - fn identity(&self) -> &str { - self.input.identity() - } } diff --git a/src/stream/src/executor/wrapper/epoch_check.rs b/src/stream/src/executor/wrapper/epoch_check.rs index 8bc785de85036..0110d8a0d86e6 100644 --- a/src/stream/src/executor/wrapper/epoch_check.rs +++ b/src/stream/src/executor/wrapper/epoch_check.rs @@ -81,18 +81,18 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; #[tokio::test] async fn test_epoch_ok() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(Default::default(), vec![]); tx.push_barrier(test_epoch(1), false); tx.push_chunk(StreamChunk::default()); tx.push_barrier(test_epoch(2), false); tx.push_barrier(test_epoch(3), false); tx.push_barrier(test_epoch(4), false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == test_epoch(1)); @@ -105,14 +105,15 @@ mod tests { #[should_panic] #[tokio::test] async fn test_epoch_bad() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(Default::default(), vec![]); tx.push_barrier(test_epoch(100), false); tx.push_chunk(StreamChunk::default()); tx.push_barrier(test_epoch(514), false); tx.push_barrier(test_epoch(514), false); tx.push_barrier(test_epoch(114), false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Barrier(b) if b.epoch.curr == test_epoch(100)); @@ -126,11 +127,12 @@ mod tests { #[should_panic] #[tokio::test] async fn test_epoch_first_not_barrier() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); tx.push_barrier(test_epoch(114), false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -138,9 +140,11 @@ mod tests { #[tokio::test] async fn test_empty() { - let (_, mut source) = MockSource::channel(Default::default(), vec![]); - source = source.stop_on_finish(false); - let checked = epoch_check(source.info().into(), source.boxed().execute()); + let (_, source) = MockSource::channel(); + let source = source + .stop_on_finish(false) + .into_executor(Default::default(), vec![]); + let checked = epoch_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert!(checked.next().await.transpose().unwrap().is_none()); diff --git a/src/stream/src/executor/wrapper/schema_check.rs b/src/stream/src/executor/wrapper/schema_check.rs index 30e665fdbd901..f0d9ed2fde2ce 100644 --- a/src/stream/src/executor/wrapper/schema_check.rs +++ b/src/stream/src/executor/wrapper/schema_check.rs @@ -63,7 +63,6 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; #[tokio::test] async fn test_schema_ok() { @@ -74,7 +73,8 @@ mod tests { ], }; - let (mut tx, source) = MockSource::channel(schema, vec![1]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![1]); tx.push_chunk(StreamChunk::from_pretty( " I F + 100 200.0 @@ -83,7 +83,7 @@ mod tests { )); tx.push_barrier(test_epoch(1), false); - let checked = schema_check(source.info().into(), source.boxed().execute()); + let checked = schema_check(source.info().clone().into(), source.execute()); pin_mut!(checked); assert_matches!(checked.next().await.unwrap().unwrap(), Message::Chunk(_)); @@ -100,7 +100,8 @@ mod tests { ], }; - let (mut tx, source) = MockSource::channel(schema, vec![1]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![1]); tx.push_chunk(StreamChunk::from_pretty( " I I + 100 200 @@ -109,7 +110,7 @@ mod tests { )); tx.push_barrier(test_epoch(1), false); - let checked = schema_check(source.info().into(), source.boxed().execute()); + let checked = schema_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); } diff --git a/src/stream/src/executor/wrapper/trace.rs b/src/stream/src/executor/wrapper/trace.rs index b480d0934a96f..df594194966be 100644 --- a/src/stream/src/executor/wrapper/trace.rs +++ b/src/stream/src/executor/wrapper/trace.rs @@ -40,7 +40,8 @@ pub async fn trace( tracing::info_span!( "executor", "otel.name" = span_name, - "actor_id" = actor_ctx.id + "message" = tracing::field::Empty, // record later + "chunk_size" = tracing::field::Empty, // record later ) }; let mut span = new_span(); @@ -48,57 +49,58 @@ pub async fn trace( pin_mut!(input); while let Some(message) = input.next().instrument(span.clone()).await.transpose()? { - // Trace the message in the span's scope. - span.in_scope(|| match &message { + // Emit a debug event and record the message type. + match &message { Message::Chunk(chunk) => { - if chunk.cardinality() > 0 { - if enable_executor_row_count { - actor_ctx - .streaming_metrics - .executor_row_count - .with_label_values(&[&actor_id_str, &fragment_id_str, &info.identity]) - .inc_by(chunk.cardinality() as u64); - } - tracing::debug!( - target: "events::stream::message::chunk", - cardinality = chunk.cardinality(), - capacity = chunk.capacity(), - "\n{}\n", chunk.to_pretty_with_schema(&info.schema), - ); + if enable_executor_row_count { + actor_ctx + .streaming_metrics + .executor_row_count + .with_label_values(&[&actor_id_str, &fragment_id_str, &info.identity]) + .inc_by(chunk.cardinality() as u64); } + tracing::debug!( + target: "events::stream::message::chunk", + parent: &span, + cardinality = chunk.cardinality(), + capacity = chunk.capacity(), + "\n{}\n", chunk.to_pretty_with_schema(&info.schema), + ); + span.record("message", "chunk"); + span.record("chunk_size", chunk.cardinality()); } Message::Watermark(watermark) => { tracing::debug!( target: "events::stream::message::watermark", + parent: &span, value = ?watermark.val, col_idx = watermark.col_idx, ); + span.record("message", "watermark"); } Message::Barrier(barrier) => { tracing::debug!( target: "events::stream::message::barrier", + parent: &span, prev_epoch = barrier.epoch.prev, curr_epoch = barrier.epoch.curr, kind = ?barrier.kind, ); + span.record("message", "barrier"); } - }); + }; - // Yield the message and update the span. - match &message { - Message::Chunk(_) | Message::Watermark(_) => yield message, - Message::Barrier(_) => { - // Drop the span as the inner executor has finished processing the barrier (then all - // data from the previous epoch). - let _ = std::mem::replace(&mut span, Span::none()); + // Drop the span as the inner executor has yielded a new message. + // + // This is essentially similar to `.instrument(new_span())`, but it allows us to + // emit the debug event and record the message type. + let _ = std::mem::replace(&mut span, Span::none()); - yield message; + yield message; - // Create a new span after we're called again. Now we're in a new epoch and the - // parent of the span is updated. - span = new_span(); - } - } + // Create a new span after we're called again. The parent span may also have been + // updated. + span = new_span(); } } diff --git a/src/stream/src/executor/wrapper/update_check.rs b/src/stream/src/executor/wrapper/update_check.rs index 4049960845282..37a2ca4f2a942 100644 --- a/src/stream/src/executor/wrapper/update_check.rs +++ b/src/stream/src/executor/wrapper/update_check.rs @@ -62,12 +62,12 @@ mod tests { use super::*; use crate::executor::test_utils::MockSource; - use crate::executor::Executor; #[should_panic] #[tokio::test] async fn test_not_next_to_each_other() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U- 114 @@ -76,7 +76,7 @@ mod tests { U+ 810", )); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -85,13 +85,14 @@ mod tests { #[should_panic] #[tokio::test] async fn test_first_one_update_insert() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U+ 114", )); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -100,7 +101,8 @@ mod tests { #[should_panic] #[tokio::test] async fn test_last_one_update_delete() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::from_pretty( " I U- 114 @@ -108,7 +110,7 @@ mod tests { U- 1919810", )); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); // should panic @@ -116,10 +118,11 @@ mod tests { #[tokio::test] async fn test_empty_chunk() { - let (mut tx, source) = MockSource::channel(Default::default(), vec![]); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(Default::default(), vec![]); tx.push_chunk(StreamChunk::default()); - let checked = update_check(source.info().into(), source.boxed().execute()); + let checked = update_check(source.info().clone().into(), source.execute()); pin_mut!(checked); checked.next().await.unwrap().unwrap(); diff --git a/src/stream/src/from_proto/append_only_dedup.rs b/src/stream/src/from_proto/append_only_dedup.rs index cdbc86683dc4f..020d13f3ff96b 100644 --- a/src/stream/src/from_proto/append_only_dedup.rs +++ b/src/stream/src/from_proto/append_only_dedup.rs @@ -20,8 +20,8 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{AppendOnlyDedupExecutor, BoxedExecutor}; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::{AppendOnlyDedupExecutor, Executor}; +use crate::task::ExecutorParams; pub struct AppendOnlyDedupExecutorBuilder; @@ -32,19 +32,19 @@ impl ExecutorBuilder for AppendOnlyDedupExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let table = node.get_state_table()?; let vnodes = params.vnode_bitmap.map(Arc::new); let state_table = StateTable::from_table_catalog(table, store, vnodes).await; - Ok(Box::new(AppendOnlyDedupExecutor::new( + let exec = AppendOnlyDedupExecutor::new( + params.actor_context, input, + params.info.pk_indices.clone(), /* TODO(rc): should change to use `dedup_column_indices`, but need to check backward compatibility */ state_table, - params.info, - params.actor_context, - stream.get_watermark_epoch(), - stream.streaming_metrics.clone(), - ))) + params.watermark_epoch, + params.executor_stats.clone(), + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/barrier_recv.rs b/src/stream/src/from_proto/barrier_recv.rs index 8d834642147f7..b8c38f0f53d32 100644 --- a/src/stream/src/from_proto/barrier_recv.rs +++ b/src/stream/src/from_proto/barrier_recv.rs @@ -27,19 +27,18 @@ impl ExecutorBuilder for BarrierRecvExecutorBuilder { params: ExecutorParams, _node: &Self::Node, _store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { assert!( params.input.is_empty(), "barrier receiver should not have input" ); let (sender, barrier_receiver) = unbounded_channel(); - stream - .context - .barrier_manager() + params + .local_barrier_manager .register_sender(params.actor_context.id, sender); - Ok(BarrierRecvExecutor::new(params.actor_context, params.info, barrier_receiver).boxed()) + let exec = BarrierRecvExecutor::new(params.actor_context, barrier_receiver); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/batch_query.rs b/src/stream/src/from_proto/batch_query.rs index 30112807266c9..5e86d7e6d5b30 100644 --- a/src/stream/src/from_proto/batch_query.rs +++ b/src/stream/src/from_proto/batch_query.rs @@ -30,13 +30,12 @@ impl ExecutorBuilder for BatchQueryExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { if node.table_desc.is_none() { // used in sharing cdc source backfill as a dummy batch plan node let mut info = params.info; info.identity = "DummyBatchQueryExecutor".to_string(); - return Ok(Box::new(DummyExecutor::new(info))); + return Ok((info, DummyExecutor).into()); } let table_desc: &StorageTableDesc = node.get_table_desc()?; @@ -56,9 +55,11 @@ impl ExecutorBuilder for BatchQueryExecutorBuilder { ); assert_eq!(table.schema().data_types(), params.info.schema.data_types()); - let executor = - BatchQueryExecutor::new(table, stream.config.developer.chunk_size, params.info); - - Ok(executor.boxed()) + let exec = BatchQueryExecutor::new( + table, + params.env.config().developer.chunk_size, + params.info.schema.clone(), + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/cdc_filter.rs b/src/stream/src/from_proto/cdc_filter.rs index 4add7360b7db7..673f6a1c5e712 100644 --- a/src/stream/src/from_proto/cdc_filter.rs +++ b/src/stream/src/from_proto/cdc_filter.rs @@ -28,12 +28,12 @@ impl ExecutorBuilder for CdcFilterExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let search_condition = build_non_strict_from_prost(node.get_search_condition()?, params.eval_error_report)?; - Ok(FilterExecutor::new(params.actor_context, params.info, input, search_condition).boxed()) + let exec = FilterExecutor::new(params.actor_context, input, search_condition); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/dml.rs b/src/stream/src/from_proto/dml.rs index e039b59016d4e..cc8836890ce7f 100644 --- a/src/stream/src/from_proto/dml.rs +++ b/src/stream/src/from_proto/dml.rs @@ -20,8 +20,8 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::error::StreamResult; use crate::executor::dml::DmlExecutor; -use crate::executor::BoxedExecutor; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::Executor; +use crate::task::ExecutorParams; pub struct DmlExecutorBuilder; @@ -32,20 +32,19 @@ impl ExecutorBuilder for DmlExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream_manager: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); let table_id = TableId::new(node.table_id); let column_descs = node.column_descs.iter().map(Into::into).collect_vec(); - Ok(Box::new(DmlExecutor::new( - params.info, + let exec = DmlExecutor::new( upstream, params.env.dml_manager_ref(), table_id, node.table_version_id, column_descs, params.env.config().developer.chunk_size, - ))) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/dynamic_filter.rs b/src/stream/src/from_proto/dynamic_filter.rs index a4bdc63f37365..be744354c3d14 100644 --- a/src/stream/src/from_proto/dynamic_filter.rs +++ b/src/stream/src/from_proto/dynamic_filter.rs @@ -32,16 +32,11 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [source_l, source_r]: [_; 2] = params.input.try_into().unwrap(); let key_l = node.get_left_key() as usize; - let vnodes = Arc::new( - params - .vnode_bitmap - .expect("vnodes not set for dynamic filter"), - ); + let vnodes = params.vnode_bitmap.map(Arc::new); let prost_condition = node.get_condition()?; let comparator = prost_condition.get_function_type()?; @@ -63,17 +58,14 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { let left_table = node.get_left_table()?; let cleaned_by_watermark = left_table.get_cleaned_by_watermark(); - if cleaned_by_watermark { - let state_table_l = WatermarkCacheStateTable::from_table_catalog( - node.get_left_table()?, - store, - Some(vnodes), - ) - .await; + let exec = if cleaned_by_watermark { + let state_table_l = + WatermarkCacheStateTable::from_table_catalog(node.get_left_table()?, store, vnodes) + .await; - Ok(Box::new(DynamicFilterExecutor::new( + DynamicFilterExecutor::new( params.actor_context, - params.info, + ¶ms.info, source_l, source_r, key_l, @@ -84,14 +76,15 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { params.env.config().developer.chunk_size, condition_always_relax, cleaned_by_watermark, - ))) + ) + .boxed() } else { let state_table_l = - StateTable::from_table_catalog(node.get_left_table()?, store, Some(vnodes)).await; + StateTable::from_table_catalog(node.get_left_table()?, store, vnodes).await; - Ok(Box::new(DynamicFilterExecutor::new( + DynamicFilterExecutor::new( params.actor_context, - params.info, + ¶ms.info, source_l, source_r, key_l, @@ -102,7 +95,10 @@ impl ExecutorBuilder for DynamicFilterExecutorBuilder { params.env.config().developer.chunk_size, condition_always_relax, cleaned_by_watermark, - ))) - } + ) + .boxed() + }; + + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/eowc_over_window.rs b/src/stream/src/from_proto/eowc_over_window.rs index 7ba9aceaa93f6..4f6e873d7bcf3 100644 --- a/src/stream/src/from_proto/eowc_over_window.rs +++ b/src/stream/src/from_proto/eowc_over_window.rs @@ -21,10 +21,8 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{ - BoxedExecutor, EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Executor, -}; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs, Executor}; +use crate::task::ExecutorParams; pub struct EowcOverWindowExecutorBuilder; @@ -35,8 +33,7 @@ impl ExecutorBuilder for EowcOverWindowExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let calls: Vec<_> = node .get_calls() @@ -57,18 +54,18 @@ impl ExecutorBuilder for EowcOverWindowExecutorBuilder { let state_table = StateTable::from_table_catalog_inconsistent_op(node.get_state_table()?, store, vnodes) .await; - Ok(EowcOverWindowExecutor::new(EowcOverWindowExecutorArgs { + let exec = EowcOverWindowExecutor::new(EowcOverWindowExecutorArgs { actor_ctx: params.actor_context, - info: params.info, input, + schema: params.info.schema.clone(), calls, partition_key_indices, order_key_index, state_table, - watermark_epoch: stream.get_watermark_epoch(), - }) - .boxed()) + watermark_epoch: params.watermark_epoch, + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/expand.rs b/src/stream/src/from_proto/expand.rs index d2eb7c581afce..34d4313e24cb7 100644 --- a/src/stream/src/from_proto/expand.rs +++ b/src/stream/src/from_proto/expand.rs @@ -26,8 +26,7 @@ impl ExecutorBuilder for ExpandExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let column_subsets = node .column_subsets @@ -40,6 +39,6 @@ impl ExecutorBuilder for ExpandExecutorBuilder { .collect_vec() }) .collect_vec(); - Ok(ExpandExecutor::new(params.info, input, column_subsets).boxed()) + Ok((params.info, ExpandExecutor::new(input, column_subsets)).into()) } } diff --git a/src/stream/src/from_proto/filter.rs b/src/stream/src/from_proto/filter.rs index d61cb177f9466..a4efb953064b8 100644 --- a/src/stream/src/from_proto/filter.rs +++ b/src/stream/src/from_proto/filter.rs @@ -27,12 +27,12 @@ impl ExecutorBuilder for FilterExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let search_condition = build_non_strict_from_prost(node.get_search_condition()?, params.eval_error_report)?; - Ok(FilterExecutor::new(params.actor_context, params.info, input, search_condition).boxed()) + let exec = FilterExecutor::new(params.actor_context, input, search_condition); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/group_top_n.rs b/src/stream/src/from_proto/group_top_n.rs index 4c73d5eeeac2a..51512cb9d94d3 100644 --- a/src/stream/src/from_proto/group_top_n.rs +++ b/src/stream/src/from_proto/group_top_n.rs @@ -14,6 +14,7 @@ use std::sync::Arc; +use risingwave_common::catalog::Schema; use risingwave_common::hash::{HashKey, HashKeyDispatcher}; use risingwave_common::types::DataType; use risingwave_common::util::sort_util::ColumnOrder; @@ -21,7 +22,7 @@ use risingwave_pb::stream_plan::GroupTopNNode; use super::*; use crate::common::table::state_table::StateTable; -use crate::executor::{ActorContextRef, AppendOnlyGroupTopNExecutor, GroupTopNExecutor}; +use crate::executor::{ActorContextRef, AppendOnlyGroupTopNExecutor, Executor, GroupTopNExecutor}; use crate::task::AtomicU64Ref; pub struct GroupTopNExecutorBuilder; @@ -33,8 +34,7 @@ impl ExecutorBuilder for GroupTopNExecutorBuilder StreamResult { + ) -> StreamResult { let group_by: Vec = node .get_group_key() .iter() @@ -62,26 +62,26 @@ impl ExecutorBuilder for GroupTopNExecutorBuilder { - input: BoxedExecutor, + input: Executor, ctx: ActorContextRef, - info: ExecutorInfo, + schema: Schema, storage_key: Vec, offset_and_limit: (usize, usize), order_by: Vec, @@ -95,7 +95,7 @@ struct GroupTopNExecutorDispatcherArgs { } impl HashKeyDispatcher for GroupTopNExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { macro_rules! build { @@ -103,7 +103,7 @@ impl HashKeyDispatcher for GroupTopNExecutorDispatcherArgs { Ok($excutor::::new( self.input, self.ctx, - self.info, + self.schema, self.storage_key, self.offset_and_limit, self.order_by, diff --git a/src/stream/src/from_proto/hash_agg.rs b/src/stream/src/from_proto/hash_agg.rs index d58d351f76192..3331640519617 100644 --- a/src/stream/src/from_proto/hash_agg.rs +++ b/src/stream/src/from_proto/hash_agg.rs @@ -35,7 +35,7 @@ pub struct HashAggExecutorDispatcherArgs { } impl HashKeyDispatcher for HashAggExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { Ok(HashAggExecutor::::new(self.args)?.boxed()) @@ -55,8 +55,7 @@ impl ExecutorBuilder for HashAggExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let group_key_indices = node .get_group_key() .iter() @@ -94,22 +93,22 @@ impl ExecutorBuilder for HashAggExecutorBuilder { build_distinct_dedup_table_from_proto(node.get_distinct_dedup_tables(), store, vnodes) .await; - HashAggExecutorDispatcherArgs { + let exec = HashAggExecutorDispatcherArgs { args: AggExecutorArgs { version: node.version(), input, actor_ctx: params.actor_context, - info: params.info, + info: params.info.clone(), - extreme_cache_size: stream.config.developer.unsafe_extreme_cache_size, + extreme_cache_size: params.env.config().developer.unsafe_extreme_cache_size, agg_calls, row_count_index: node.get_row_count_index() as usize, storages, intermediate_state_table, distinct_dedup_tables, - watermark_epoch: stream.get_watermark_epoch(), + watermark_epoch: params.watermark_epoch, extra: HashAggExecutorExtraArgs { group_key_indices, chunk_size: params.env.config().developer.chunk_size, @@ -123,6 +122,7 @@ impl ExecutorBuilder for HashAggExecutorBuilder { }, group_key_types, } - .dispatch() + .dispatch()?; + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/hash_join.rs b/src/stream/src/from_proto/hash_join.rs index 008ee0d2d0b13..ad69fd1ecd5ac 100644 --- a/src/stream/src/from_proto/hash_join.rs +++ b/src/stream/src/from_proto/hash_join.rs @@ -27,7 +27,7 @@ use super::*; use crate::common::table::state_table::StateTable; use crate::executor::hash_join::*; use crate::executor::monitor::StreamingMetrics; -use crate::executor::ActorContextRef; +use crate::executor::{ActorContextRef, Executor, JoinType}; use crate::task::AtomicU64Ref; pub struct HashJoinExecutorBuilder; @@ -39,8 +39,7 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let is_append_only = node.is_append_only; let vnodes = Arc::new(params.vnode_bitmap.expect("vnodes not set for hash join")); @@ -137,7 +136,7 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { let args = HashJoinExecutorDispatcherArgs { ctx: params.actor_context, - info: params.info, + info: params.info.clone(), source_l, source_r, params_l, @@ -150,7 +149,7 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { degree_state_table_l, state_table_r, degree_state_table_r, - lru_manager: stream.get_watermark_epoch(), + lru_manager: params.watermark_epoch, is_append_only, metrics: params.executor_stats, join_type_proto: node.get_join_type()?, @@ -158,15 +157,16 @@ impl ExecutorBuilder for HashJoinExecutorBuilder { chunk_size: params.env.config().developer.chunk_size, }; - args.dispatch() + let exec = args.dispatch()?; + Ok((params.info, exec).into()) } } struct HashJoinExecutorDispatcherArgs { ctx: ActorContextRef, info: ExecutorInfo, - source_l: Box, - source_r: Box, + source_l: Executor, + source_r: Executor, params_l: JoinParams, params_r: JoinParams, null_safe: Vec, @@ -186,34 +186,33 @@ struct HashJoinExecutorDispatcherArgs { } impl HashKeyDispatcher for HashJoinExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { /// This macro helps to fill the const generic type parameter. macro_rules! build { ($join_type:ident) => { - Ok(Box::new( - HashJoinExecutor::::new( - self.ctx, - self.info, - self.source_l, - self.source_r, - self.params_l, - self.params_r, - self.null_safe, - self.output_indices, - self.cond, - self.inequality_pairs, - self.state_table_l, - self.degree_state_table_l, - self.state_table_r, - self.degree_state_table_r, - self.lru_manager, - self.is_append_only, - self.metrics, - self.chunk_size, - ), - )) + Ok(HashJoinExecutor::::new( + self.ctx, + self.info, + self.source_l, + self.source_r, + self.params_l, + self.params_r, + self.null_safe, + self.output_indices, + self.cond, + self.inequality_pairs, + self.state_table_l, + self.degree_state_table_l, + self.state_table_r, + self.degree_state_table_r, + self.lru_manager, + self.is_append_only, + self.metrics, + self.chunk_size, + ) + .boxed()) }; } match self.join_type_proto { diff --git a/src/stream/src/from_proto/hop_window.rs b/src/stream/src/from_proto/hop_window.rs index b4dbe1fd010ea..2598ae927608e 100644 --- a/src/stream/src/from_proto/hop_window.rs +++ b/src/stream/src/from_proto/hop_window.rs @@ -27,17 +27,8 @@ impl ExecutorBuilder for HopWindowExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { - let ExecutorParams { - actor_context, - info, - input, - env, - .. - } = params; - - let input = input.into_iter().next().unwrap(); + ) -> StreamResult { + let input = params.input.into_iter().next().unwrap(); // TODO: reuse the schema derivation with frontend. let output_indices = node .get_output_indices() @@ -60,11 +51,10 @@ impl ExecutorBuilder for HopWindowExecutorBuilder { let window_slide = node.get_window_slide()?.into(); let window_size = node.get_window_size()?.into(); - let chunk_size = env.config().developer.chunk_size; + let chunk_size = params.env.config().developer.chunk_size; - Ok(HopWindowExecutor::new( - actor_context, - info, + let exec = HopWindowExecutor::new( + params.actor_context, input, time_col, window_slide, @@ -73,7 +63,7 @@ impl ExecutorBuilder for HopWindowExecutorBuilder { window_end_exprs, output_indices, chunk_size, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/lookup.rs b/src/stream/src/from_proto/lookup.rs index fab103294ad45..dc7f7e3c49dfe 100644 --- a/src/stream/src/from_proto/lookup.rs +++ b/src/stream/src/from_proto/lookup.rs @@ -30,8 +30,7 @@ impl ExecutorBuilder for LookupExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream_manager: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let lookup = node; let [stream, arrangement]: [_; 2] = params.input.try_into().unwrap(); @@ -70,9 +69,9 @@ impl ExecutorBuilder for LookupExecutorBuilder { table_desc, ); - Ok(Box::new(LookupExecutor::new(LookupExecutorParams { + let exec = LookupExecutor::new(LookupExecutorParams { ctx: params.actor_context, - info: params.info, + info: params.info.clone(), arrangement, stream, arrangement_col_descs, @@ -82,8 +81,9 @@ impl ExecutorBuilder for LookupExecutorBuilder { arrange_join_key_indices: lookup.arrange_key.iter().map(|x| *x as usize).collect(), column_mapping: lookup.column_mapping.iter().map(|x| *x as usize).collect(), storage_table, - watermark_epoch: stream_manager.get_watermark_epoch(), + watermark_epoch: params.watermark_epoch, chunk_size: params.env.config().developer.chunk_size, - }))) + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/lookup_union.rs b/src/stream/src/from_proto/lookup_union.rs index 9ff765a96d3ed..93e847609cae2 100644 --- a/src/stream/src/from_proto/lookup_union.rs +++ b/src/stream/src/from_proto/lookup_union.rs @@ -26,8 +26,8 @@ impl ExecutorBuilder for LookupUnionExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { - Ok(LookupUnionExecutor::new(params.info, params.input, node.order.clone()).boxed()) + ) -> StreamResult { + let exec = LookupUnionExecutor::new(params.input, node.order.clone()); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/merge.rs b/src/stream/src/from_proto/merge.rs index 6259aba71acc6..eded1f59d294e 100644 --- a/src/stream/src/from_proto/merge.rs +++ b/src/stream/src/from_proto/merge.rs @@ -27,8 +27,7 @@ impl ExecutorBuilder for MergeExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let upstreams = node.get_upstream_actor_id(); let upstream_fragment_id = node.get_upstream_fragment_id(); @@ -36,8 +35,8 @@ impl ExecutorBuilder for MergeExecutorBuilder { .iter() .map(|&upstream_actor_id| { new_input( - &stream.context, - stream.streaming_metrics.clone(), + ¶ms.shared_context, + params.executor_stats.clone(), params.actor_context.id, params.fragment_id, upstream_actor_id, @@ -57,30 +56,29 @@ impl ExecutorBuilder for MergeExecutorBuilder { DispatcherType::NoShuffle => true, }; - if always_single_input { - Ok(ReceiverExecutor::new( + let exec = if always_single_input { + ReceiverExecutor::new( params.actor_context, - params.info, params.fragment_id, upstream_fragment_id, inputs.into_iter().exactly_one().unwrap(), - stream.context.clone(), + params.shared_context.clone(), params.operator_id, - stream.streaming_metrics.clone(), + params.executor_stats.clone(), ) - .boxed()) + .boxed() } else { - Ok(MergeExecutor::new( + MergeExecutor::new( params.actor_context, - params.info, params.fragment_id, upstream_fragment_id, inputs, - stream.context.clone(), + params.shared_context.clone(), params.operator_id, - stream.streaming_metrics.clone(), + params.executor_stats.clone(), ) - .boxed()) - } + .boxed() + }; + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/mod.rs b/src/stream/src/from_proto/mod.rs index 9a9e83c0a328f..f6b68cd8e915a 100644 --- a/src/stream/src/from_proto/mod.rs +++ b/src/stream/src/from_proto/mod.rs @@ -45,6 +45,7 @@ mod source; mod stateless_simple_agg; mod stream_cdc_scan; mod stream_scan; +mod subscription; mod temporal_join; mod top_n; mod union; @@ -92,28 +93,28 @@ use self::top_n::*; use self::union::*; use self::watermark_filter::WatermarkFilterBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Executor, ExecutorInfo}; +use crate::executor::{Execute, Executor, ExecutorInfo}; +use crate::from_proto::subscription::SubscriptionExecutorBuilder; use crate::from_proto::values::ValuesExecutorBuilder; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::task::ExecutorParams; trait ExecutorBuilder { type Node; - /// Create a [`BoxedExecutor`] from [`StreamNode`]. - fn new_boxed_executor( + /// Create an [`Executor`] from [`StreamNode`]. + async fn new_boxed_executor( params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> impl std::future::Future> + Send; + ) -> StreamResult; } macro_rules! build_executor { - ($source:expr, $node:expr, $store:expr, $stream:expr, $($proto_type_name:path => $data_type:ty),* $(,)?) => { + ($source:expr, $node:expr, $store:expr, $($proto_type_name:path => $data_type:ty),* $(,)?) => { match $node.get_node_body().unwrap() { $( $proto_type_name(node) => { - <$data_type>::new_boxed_executor($source, node, $store, $stream).await + <$data_type>::new_boxed_executor($source, node, $store).await }, )* NodeBody::Exchange(_) | NodeBody::DeltaIndexJoin(_) => unreachable!() @@ -124,15 +125,13 @@ macro_rules! build_executor { /// Create an executor from protobuf [`StreamNode`]. pub async fn create_executor( params: ExecutorParams, - stream: &mut LocalStreamManagerCore, node: &StreamNode, store: impl StateStore, -) -> StreamResult { +) -> StreamResult { build_executor! { params, node, store, - stream, NodeBody::Source => SourceExecutorBuilder, NodeBody::Sink => SinkExecutorBuilder, NodeBody::Project => ProjectExecutorBuilder, @@ -148,6 +147,7 @@ pub async fn create_executor( NodeBody::BatchPlan => BatchQueryExecutorBuilder, NodeBody::Merge => MergeExecutorBuilder, NodeBody::Materialize => MaterializeExecutorBuilder, + NodeBody::Subscription => SubscriptionExecutorBuilder, NodeBody::Filter => FilterExecutorBuilder, NodeBody::CdcFilter => CdcFilterExecutorBuilder, NodeBody::Arrange => ArrangeExecutorBuilder, diff --git a/src/stream/src/from_proto/mview.rs b/src/stream/src/from_proto/mview.rs index 153108d00a916..80aab576435ca 100644 --- a/src/stream/src/from_proto/mview.rs +++ b/src/stream/src/from_proto/mview.rs @@ -32,8 +32,7 @@ impl ExecutorBuilder for MaterializeExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let order_key = node @@ -52,28 +51,28 @@ impl ExecutorBuilder for MaterializeExecutorBuilder { ($SD:ident) => { MaterializeExecutor::<_, $SD>::new( input, - params.info, + params.info.schema.clone(), store, order_key, params.actor_context, params.vnode_bitmap.map(Arc::new), table, - stream.get_watermark_epoch(), + params.watermark_epoch, conflict_behavior, - stream.streaming_metrics.clone(), + params.executor_stats.clone(), ) .await .boxed() }; } - let executor = if versioned { + let exec = if versioned { new_executor!(ColumnAwareSerde) } else { new_executor!(BasicSerde) }; - Ok(executor) + Ok((params.info, exec).into()) } } @@ -86,8 +85,7 @@ impl ExecutorBuilder for ArrangeExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let keys = node @@ -104,20 +102,20 @@ impl ExecutorBuilder for ArrangeExecutorBuilder { let vnodes = params.vnode_bitmap.map(Arc::new); let conflict_behavior = ConflictBehavior::from_protobuf(&table.handle_pk_conflict_behavior()); - let executor = MaterializeExecutor::<_, BasicSerde>::new( + let exec = MaterializeExecutor::<_, BasicSerde>::new( input, - params.info, + params.info.schema.clone(), store, keys, params.actor_context, vnodes, table, - stream.get_watermark_epoch(), + params.watermark_epoch, conflict_behavior, - stream.streaming_metrics.clone(), + params.executor_stats.clone(), ) .await; - Ok(executor.boxed()) + Ok((params.info, exec.boxed()).into()) } } diff --git a/src/stream/src/from_proto/no_op.rs b/src/stream/src/from_proto/no_op.rs index 73dacf10043c3..31494d771b903 100644 --- a/src/stream/src/from_proto/no_op.rs +++ b/src/stream/src/from_proto/no_op.rs @@ -17,8 +17,8 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Executor, NoOpExecutor}; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::{Executor, NoOpExecutor}; +use crate::task::ExecutorParams; pub struct NoOpExecutorBuilder; @@ -29,9 +29,8 @@ impl ExecutorBuilder for NoOpExecutorBuilder { params: ExecutorParams, _node: &NoOpNode, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); - Ok(NoOpExecutor::new(params.actor_context, params.info, input).boxed()) + Ok((params.info, NoOpExecutor::new(params.actor_context, input)).into()) } } diff --git a/src/stream/src/from_proto/now.rs b/src/stream/src/from_proto/now.rs index 601c1ec3ad585..06de0cacb7197 100644 --- a/src/stream/src/from_proto/now.rs +++ b/src/stream/src/from_proto/now.rs @@ -19,8 +19,8 @@ use tokio::sync::mpsc::unbounded_channel; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, NowExecutor}; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::{Executor, NowExecutor}; +use crate::task::ExecutorParams; pub struct NowExecutorBuilder; @@ -31,21 +31,20 @@ impl ExecutorBuilder for NowExecutorBuilder { params: ExecutorParams, node: &NowNode, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let (sender, barrier_receiver) = unbounded_channel(); - stream - .context - .barrier_manager() + params + .local_barrier_manager .register_sender(params.actor_context.id, sender); let state_table = StateTable::from_table_catalog(node.get_state_table()?, store, None).await; - Ok(Box::new(NowExecutor::new( - params.info, + let exec = NowExecutor::new( + params.info.schema.data_types(), barrier_receiver, state_table, - ))) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/over_window.rs b/src/stream/src/from_proto/over_window.rs index c01e01824fe12..f7ca73c183a81 100644 --- a/src/stream/src/from_proto/over_window.rs +++ b/src/stream/src/from_proto/over_window.rs @@ -23,8 +23,8 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::common::table::state_table::StateTable; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, Executor, OverWindowExecutor, OverWindowExecutorArgs}; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::{Executor, OverWindowExecutor, OverWindowExecutorArgs}; +use crate::task::ExecutorParams; pub struct OverWindowExecutorBuilder; @@ -35,8 +35,7 @@ impl ExecutorBuilder for OverWindowExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let calls: Vec<_> = node .get_calls() @@ -61,26 +60,26 @@ impl ExecutorBuilder for OverWindowExecutorBuilder { )); let state_table = StateTable::from_table_catalog(node.get_state_table()?, store, vnodes).await; - Ok(OverWindowExecutor::new(OverWindowExecutorArgs { + let exec = OverWindowExecutor::new(OverWindowExecutorArgs { actor_ctx: params.actor_context, - info: params.info, input, + schema: params.info.schema.clone(), calls, partition_key_indices, order_key_indices, order_key_order_types, state_table, - watermark_epoch: stream.get_watermark_epoch(), + watermark_epoch: params.watermark_epoch, metrics: params.executor_stats, chunk_size: params.env.config().developer.chunk_size, cache_policy: OverWindowCachePolicy::from_protobuf( node.get_cache_policy().unwrap_or_default(), ), - }) - .boxed()) + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/project.rs b/src/stream/src/from_proto/project.rs index 9fd1176daffdc..d7f96c4dffcbf 100644 --- a/src/stream/src/from_proto/project.rs +++ b/src/stream/src/from_proto/project.rs @@ -30,8 +30,7 @@ impl ExecutorBuilder for ProjectExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let project_exprs: Vec<_> = node .get_select_list() @@ -61,15 +60,14 @@ impl ExecutorBuilder for ProjectExecutorBuilder { ) }); let materialize_selectivity_threshold = if extremely_light { 0.0 } else { 0.5 }; - Ok(ProjectExecutor::new( + let exec = ProjectExecutor::new( params.actor_context, - params.info, input, project_exprs, watermark_derivations, nondecreasing_expr_indices, materialize_selectivity_threshold, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/project_set.rs b/src/stream/src/from_proto/project_set.rs index b4879e7255353..c2338394b33ef 100644 --- a/src/stream/src/from_proto/project_set.rs +++ b/src/stream/src/from_proto/project_set.rs @@ -29,8 +29,7 @@ impl ExecutorBuilder for ProjectSetExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let select_list: Vec<_> = node .get_select_list() @@ -56,15 +55,14 @@ impl ExecutorBuilder for ProjectSetExecutorBuilder { .collect(); let chunk_size = params.env.config().developer.chunk_size; - Ok(ProjectSetExecutor::new( + let exec = ProjectSetExecutor::new( params.actor_context, - params.info, input, select_list, chunk_size, watermark_derivations, nondecreasing_expr_indices, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/row_id_gen.rs b/src/stream/src/from_proto/row_id_gen.rs index 4996ac14268f2..ab87566547ac6 100644 --- a/src/stream/src/from_proto/row_id_gen.rs +++ b/src/stream/src/from_proto/row_id_gen.rs @@ -18,8 +18,8 @@ use risingwave_storage::StateStore; use super::ExecutorBuilder; use crate::error::StreamResult; use crate::executor::row_id_gen::RowIdGenExecutor; -use crate::executor::BoxedExecutor; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::Executor; +use crate::task::ExecutorParams; pub struct RowIdGenExecutorBuilder; @@ -30,20 +30,18 @@ impl ExecutorBuilder for RowIdGenExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); tracing::debug!("row id gen executor: {:?}", params.vnode_bitmap); let vnodes = params .vnode_bitmap .expect("vnodes not set for row id gen executor"); - let executor = RowIdGenExecutor::new( + let exec = RowIdGenExecutor::new( params.actor_context, - params.info, upstream, node.row_id_index as _, vnodes, ); - Ok(Box::new(executor)) + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/simple_agg.rs b/src/stream/src/from_proto/simple_agg.rs index 1bca5e18c67d0..16809edb8bcaf 100644 --- a/src/stream/src/from_proto/simple_agg.rs +++ b/src/stream/src/from_proto/simple_agg.rs @@ -34,8 +34,7 @@ impl ExecutorBuilder for SimpleAggExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let agg_calls: Vec = node .get_agg_calls() @@ -56,23 +55,24 @@ impl ExecutorBuilder for SimpleAggExecutorBuilder { build_distinct_dedup_table_from_proto(node.get_distinct_dedup_tables(), store, None) .await; - Ok(SimpleAggExecutor::new(AggExecutorArgs { + let exec = SimpleAggExecutor::new(AggExecutorArgs { version: node.version(), input, actor_ctx: params.actor_context, - info: params.info, + info: params.info.clone(), - extreme_cache_size: stream.config.developer.unsafe_extreme_cache_size, + extreme_cache_size: params.env.config().developer.unsafe_extreme_cache_size, agg_calls, row_count_index: node.get_row_count_index() as usize, storages, intermediate_state_table, distinct_dedup_tables, - watermark_epoch: stream.get_watermark_epoch(), + watermark_epoch: params.watermark_epoch, extra: SimpleAggExecutorExtraArgs {}, - })? - .boxed()) + })?; + + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/sink.rs b/src/stream/src/from_proto/sink.rs index 659450e83c077..884925d4a35ab 100644 --- a/src/stream/src/from_proto/sink.rs +++ b/src/stream/src/from_proto/sink.rs @@ -15,21 +15,87 @@ use std::sync::Arc; use anyhow::anyhow; -use risingwave_common::catalog::ColumnCatalog; +use risingwave_common::catalog::{ColumnCatalog, Schema}; +use risingwave_common::types::DataType; use risingwave_connector::match_sink_name_str; use risingwave_connector::sink::catalog::{SinkFormatDesc, SinkType}; use risingwave_connector::sink::{ - SinkError, SinkParam, SinkWriterParam, CONNECTOR_TYPE_KEY, SINK_TYPE_OPTION, + SinkError, SinkMetaClient, SinkParam, SinkWriterParam, CONNECTOR_TYPE_KEY, SINK_TYPE_OPTION, }; +use risingwave_pb::catalog::Table; +use risingwave_pb::plan_common::PbColumnCatalog; use risingwave_pb::stream_plan::{SinkLogStoreType, SinkNode}; use super::*; use crate::common::log_store_impl::in_mem::BoundedInMemLogStoreFactory; -use crate::common::log_store_impl::kv_log_store::{KvLogStoreFactory, KvLogStoreMetrics}; +use crate::common::log_store_impl::kv_log_store::{ + KvLogStoreFactory, KvLogStoreMetrics, KvLogStorePkInfo, KV_LOG_STORE_V2_INFO, +}; use crate::executor::SinkExecutor; pub struct SinkExecutorBuilder; +fn resolve_pk_info( + input_schema: &Schema, + log_store_table: &Table, +) -> StreamResult<&'static KvLogStorePkInfo> { + let predefined_column_len = log_store_table.columns.len() - input_schema.fields.len(); + + #[expect(deprecated)] + let info = match predefined_column_len { + len if len + == crate::common::log_store_impl::kv_log_store::KV_LOG_STORE_V1_INFO + .predefined_column_len() => + { + Ok(&crate::common::log_store_impl::kv_log_store::KV_LOG_STORE_V1_INFO) + } + len if len == KV_LOG_STORE_V2_INFO.predefined_column_len() => Ok(&KV_LOG_STORE_V2_INFO), + other_len => Err(anyhow!( + "invalid log store predefined len {}. log store table: {:?}, input_schema: {:?}", + other_len, + log_store_table, + input_schema + )), + }?; + validate_payload_schema( + &log_store_table.columns[predefined_column_len..], + input_schema, + )?; + Ok(info) +} + +fn validate_payload_schema( + log_store_payload_schema: &[PbColumnCatalog], + input_schema: &Schema, +) -> StreamResult<()> { + if log_store_payload_schema + .iter() + .zip_eq(input_schema.fields.iter()) + .map(|(log_store_col, input_field)| { + let log_store_col_type = DataType::from( + log_store_col + .column_desc + .as_ref() + .unwrap() + .column_type + .as_ref() + .unwrap(), + ); + log_store_col_type.equals_datatype(&input_field.data_type) + }) + .all(|equal| equal) + { + Ok(()) + } else { + Err(anyhow!( + "mismatch schema: log store: {:?}, input: {:?}", + log_store_payload_schema, + input_schema + ) + .into()) + } +} + impl ExecutorBuilder for SinkExecutorBuilder { type Node = SinkNode; @@ -37,8 +103,7 @@ impl ExecutorBuilder for SinkExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input_executor]: [_; 1] = params.input.try_into().unwrap(); let sink_desc = node.sink_desc.as_ref().unwrap(); @@ -104,7 +169,7 @@ impl ExecutorBuilder for SinkExecutorBuilder { let sink_id_str = format!("{}", sink_id.sink_id); - let sink_metrics = stream.streaming_metrics.new_sink_metrics( + let sink_metrics = params.executor_stats.new_sink_metrics( ¶ms.info.identity, sink_id_str.as_str(), connector, @@ -114,8 +179,9 @@ impl ExecutorBuilder for SinkExecutorBuilder { connector_params: params.env.connector_params(), executor_id: params.executor_id, vnode_bitmap: params.vnode_bitmap.clone(), - meta_client: params.env.meta_client(), + meta_client: params.env.meta_client().map(SinkMetaClient::MetaClient), sink_metrics, + extra_partition_col_idx: sink_desc.extra_partition_col_idx.map(|v| v as usize), }; let log_store_identity = format!( @@ -123,23 +189,22 @@ impl ExecutorBuilder for SinkExecutorBuilder { connector, sink_id.sink_id, params.executor_id ); - match node.log_store_type() { + let exec = match node.log_store_type() { // Default value is the normal in memory log store to be backward compatible with the // previously unset value SinkLogStoreType::InMemoryLogStore | SinkLogStoreType::Unspecified => { let factory = BoundedInMemLogStoreFactory::new(1); - Ok(Box::new( - SinkExecutor::new( - params.actor_context, - params.info, - input_executor, - sink_write_param, - sink_param, - columns, - factory, - ) - .await?, - )) + SinkExecutor::new( + params.actor_context, + params.info.clone(), + input_executor, + sink_write_param, + sink_param, + columns, + factory, + ) + .await? + .boxed() } SinkLogStoreType::KvLogStore => { let metrics = KvLogStoreMetrics::new( @@ -148,29 +213,36 @@ impl ExecutorBuilder for SinkExecutorBuilder { &sink_param, connector, ); + + let table = node.table.as_ref().unwrap().clone(); + let input_schema = input_executor.schema(); + let pk_info = resolve_pk_info(input_schema, &table)?; + // TODO: support setting max row count in config let factory = KvLogStoreFactory::new( state_store, - node.table.as_ref().unwrap().clone(), + table, params.vnode_bitmap.clone().map(Arc::new), 65536, metrics, log_store_identity, + pk_info, ); - Ok(Box::new( - SinkExecutor::new( - params.actor_context, - params.info, - input_executor, - sink_write_param, - sink_param, - columns, - factory, - ) - .await?, - )) + SinkExecutor::new( + params.actor_context, + params.info.clone(), + input_executor, + sink_write_param, + sink_param, + columns, + factory, + ) + .await? + .boxed() } - } + }; + + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/sort.rs b/src/stream/src/from_proto/sort.rs index e19a657e227d9..e901d66e5d8ff 100644 --- a/src/stream/src/from_proto/sort.rs +++ b/src/stream/src/from_proto/sort.rs @@ -29,19 +29,19 @@ impl ExecutorBuilder for SortExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let vnodes = Arc::new(params.vnode_bitmap.expect("vnodes not set for sort")); let state_table = StateTable::from_table_catalog(node.get_state_table()?, store, Some(vnodes)).await; - Ok(Box::new(SortExecutor::new(SortExecutorArgs { + let exec = SortExecutor::new(SortExecutorArgs { actor_ctx: params.actor_context, - info: params.info, + schema: params.info.schema.clone(), input, buffer_table: state_table, chunk_size: params.env.config().developer.chunk_size, sort_column_index: node.sort_column_index as _, - }))) + }); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/source/fs_fetch.rs b/src/stream/src/from_proto/source/fs_fetch.rs index 8d8cb78a80b19..355f7d874dec8 100644 --- a/src/stream/src/from_proto/source/fs_fetch.rs +++ b/src/stream/src/from_proto/source/fs_fetch.rs @@ -14,22 +14,22 @@ use std::sync::Arc; -use risingwave_common::catalog::{ColumnId, TableId}; +use risingwave_common::catalog::TableId; use risingwave_connector::source::filesystem::opendal_source::{ OpendalGcs, OpendalPosixFs, OpendalS3, }; +use risingwave_connector::source::reader::desc::SourceDescBuilder; use risingwave_connector::source::{ConnectorProperties, SourceCtrlOpts}; use risingwave_pb::stream_plan::StreamFsFetchNode; -use risingwave_source::source_desc::SourceDescBuilder; use risingwave_storage::StateStore; use crate::error::StreamResult; use crate::executor::{ - BoxedExecutor, Executor, FlowControlExecutor, FsFetchExecutor, SourceStateTableHandler, + Execute, Executor, FlowControlExecutor, FsFetchExecutor, SourceStateTableHandler, StreamSourceCore, }; use crate::from_proto::ExecutorBuilder; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::task::ExecutorParams; pub struct FsFetchExecutorBuilder; @@ -40,8 +40,7 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); let source = node.node_inner.as_ref().unwrap(); @@ -62,12 +61,13 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { ); let source_ctrl_opts = SourceCtrlOpts { chunk_size: params.env.config().developer.chunk_size, + rate_limit: source.rate_limit.map(|x| x as _), }; - let source_column_ids: Vec<_> = source - .columns + let source_column_ids: Vec<_> = source_desc_builder + .column_catalogs_to_source_column_descs() .iter() - .map(|column| ColumnId::from(column.get_column_desc().unwrap().column_id)) + .map(|column| column.column_id) .collect(); let vnodes = Some(Arc::new( @@ -89,11 +89,10 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { state_table_handler, ); - let executor = match properties { + let exec = match properties { risingwave_connector::source::ConnectorProperties::Gcs(_) => { FsFetchExecutor::<_, OpendalGcs>::new( params.actor_context.clone(), - params.info, stream_source_core, upstream, source_ctrl_opts, @@ -104,7 +103,6 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { risingwave_connector::source::ConnectorProperties::OpendalS3(_) => { FsFetchExecutor::<_, OpendalS3>::new( params.actor_context.clone(), - params.info, stream_source_core, upstream, source_ctrl_opts, @@ -115,7 +113,6 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { risingwave_connector::source::ConnectorProperties::PosixFs(_) => { FsFetchExecutor::<_, OpendalPosixFs>::new( params.actor_context.clone(), - params.info, stream_source_core, upstream, source_ctrl_opts, @@ -125,7 +122,11 @@ impl ExecutorBuilder for FsFetchExecutorBuilder { } _ => unreachable!(), }; + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); + let rate_limit = source.rate_limit.map(|x| x as _); - Ok(FlowControlExecutor::new(executor, params.actor_context, rate_limit).boxed()) + let exec = FlowControlExecutor::new((info, exec).into(), params.actor_context, rate_limit); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/source/trad_source.rs b/src/stream/src/from_proto/source/trad_source.rs index 5b93585aea4b1..7870bab10151b 100644 --- a/src/stream/src/from_proto/source/trad_source.rs +++ b/src/stream/src/from_proto/source/trad_source.rs @@ -13,15 +13,20 @@ // limitations under the License. use risingwave_common::catalog::{ - default_key_column_name_version_mapping, ColumnId, TableId, KAFKA_TIMESTAMP_COLUMN_NAME, + default_key_column_name_version_mapping, TableId, KAFKA_TIMESTAMP_COLUMN_NAME, +}; +use risingwave_connector::source::reader::desc::SourceDescBuilder; +use risingwave_connector::source::{ + should_copy_to_format_encode_options, ConnectorProperties, SourceCtrlOpts, UPSTREAM_SOURCE_KEY, }; -use risingwave_connector::source::{ConnectorProperties, SourceCtrlOpts}; use risingwave_pb::data::data_type::TypeName as PbTypeName; +use risingwave_pb::plan_common::additional_column::ColumnType as AdditionalColumnType; use risingwave_pb::plan_common::{ - AdditionalColumnType, ColumnDescVersion, FormatType, PbEncodeType, + AdditionalColumn, AdditionalColumnKey, AdditionalColumnTimestamp, + AdditionalColumnType as LegacyAdditionalColumnType, ColumnDescVersion, FormatType, + PbEncodeType, }; use risingwave_pb::stream_plan::SourceNode; -use risingwave_source::source_desc::SourceDescBuilder; use risingwave_storage::panic_store::PanicStateStore; use tokio::sync::mpsc::unbounded_channel; @@ -29,7 +34,7 @@ use super::*; use crate::executor::source::{FsListExecutor, StreamSourceCore}; use crate::executor::source_executor::SourceExecutor; use crate::executor::state_table_handler::SourceStateTableHandler; -use crate::executor::{FlowControlExecutor, FsSourceExecutor}; +use crate::executor::FlowControlExecutor; const FS_CONNECTORS: &[&str] = &["s3"]; pub struct SourceExecutorBuilder; @@ -41,32 +46,46 @@ impl ExecutorBuilder for SourceExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let (sender, barrier_receiver) = unbounded_channel(); - stream - .context - .barrier_manager() + params + .local_barrier_manager .register_sender(params.actor_context.id, sender); let system_params = params.env.system_params_manager_ref().get_params(); if let Some(source) = &node.source_inner { - let executor = { + let exec = { let source_id = TableId::new(source.source_id); let source_name = source.source_name.clone(); - let source_info = source.get_info()?; + let mut source_info = source.get_info()?.clone(); - let mut source_columns = source.columns.clone(); + if source_info.format_encode_options.is_empty() { + // compatible code: quick fix for , + // will move the logic to FragmentManager::init in release 1.7. + let connector = source + .with_properties + .get(UPSTREAM_SOURCE_KEY) + .unwrap_or(&String::default()) + .to_owned(); + source_info.format_encode_options.extend( + source.with_properties.iter().filter_map(|(k, v)| { + should_copy_to_format_encode_options(k, &connector) + .then_some((k.to_owned(), v.to_owned())) + }), + ); + } + let mut source_columns = source.columns.clone(); { // compatible code: introduced in https://github.com/risingwavelabs/risingwave/pull/13707 // for upsert and (avro | protobuf) overwrite the `_rw_key` column's ColumnDesc.additional_column_type to Key if source_info.format() == FormatType::Upsert && (source_info.row_encode() == PbEncodeType::Avro - || source_info.row_encode() == PbEncodeType::Protobuf) + || source_info.row_encode() == PbEncodeType::Protobuf + || source_info.row_encode() == PbEncodeType::Json) { - let _ = source_columns.iter_mut().map(|c| { - let _ = c.column_desc.as_mut().map(|desc| { + for c in &mut source_columns { + if let Some(desc) = c.column_desc.as_mut() { let is_bytea = desc .get_column_type() .map(|col_type| col_type.type_name == PbTypeName::Bytea as i32) @@ -75,20 +94,36 @@ impl ExecutorBuilder for SourceExecutorBuilder { &desc.version() ) && is_bytea - // the column is from a legacy version + // the column is from a legacy version (before v1.5.x) && desc.version == ColumnDescVersion::Unspecified as i32 { - desc.additional_column_type = AdditionalColumnType::Key as i32; + desc.additional_column = Some(AdditionalColumn { + column_type: Some(AdditionalColumnType::Key( + AdditionalColumnKey {}, + )), + }); } - }); - }); + + // the column is from a legacy version (v1.6.x) + // introduced in https://github.com/risingwavelabs/risingwave/pull/15226 + if desc.additional_column_type + == LegacyAdditionalColumnType::Key as i32 + { + desc.additional_column = Some(AdditionalColumn { + column_type: Some(AdditionalColumnType::Key( + AdditionalColumnKey {}, + )), + }); + } + } + } } } { // compatible code: handle legacy column `_rw_kafka_timestamp` // the column is auto added for all kafka source to empower batch query on source - // solution: rewrite the column `additional_column_type` to Timestamp + // solution: rewrite the column `additional_column` to Timestamp let _ = source_columns.iter_mut().map(|c| { let _ = c.column_desc.as_mut().map(|desc| { @@ -103,8 +138,11 @@ impl ExecutorBuilder for SourceExecutorBuilder { // the column is from a legacy version && desc.version == ColumnDescVersion::Unspecified as i32 { - desc.additional_column_type = - AdditionalColumnType::Timestamp as i32; + desc.additional_column = Some(AdditionalColumn { + column_type: Some(AdditionalColumnType::Timestamp( + AdditionalColumnTimestamp {}, + )), + }); } }); }); @@ -115,7 +153,7 @@ impl ExecutorBuilder for SourceExecutorBuilder { params.env.source_metrics(), source.row_id_index.map(|x| x as _), source.with_properties.clone(), - source_info.clone(), + source_info, params.env.connector_params(), params.env.config().developer.connector_message_buffer_size, // `pk_indices` is used to ensure that a message will be skipped instead of parsed @@ -132,11 +170,13 @@ impl ExecutorBuilder for SourceExecutorBuilder { let source_ctrl_opts = SourceCtrlOpts { chunk_size: params.env.config().developer.chunk_size, + rate_limit: source.rate_limit.map(|x| x as _), }; - let source_column_ids: Vec<_> = source_columns + let source_column_ids: Vec<_> = source_desc_builder + .column_catalogs_to_source_column_descs() .iter() - .map(|column| ColumnId::from(column.get_column_desc().unwrap().column_id)) + .map(|column| column.column_id) .collect(); let state_table_handler = SourceStateTableHandler::from_table_catalog( @@ -162,9 +202,9 @@ impl ExecutorBuilder for SourceExecutorBuilder { ConnectorProperties::is_new_fs_connector_hash_map(&source.with_properties); if is_fs_connector { - FsSourceExecutor::new( + #[expect(deprecated)] + crate::executor::FsSourceExecutor::new( params.actor_context.clone(), - params.info, stream_source_core, params.executor_stats, barrier_receiver, @@ -175,7 +215,6 @@ impl ExecutorBuilder for SourceExecutorBuilder { } else if is_fs_v2_connector { FsListExecutor::new( params.actor_context.clone(), - params.info.clone(), Some(stream_source_core), params.executor_stats.clone(), barrier_receiver, @@ -187,7 +226,6 @@ impl ExecutorBuilder for SourceExecutorBuilder { } else { SourceExecutor::new( params.actor_context.clone(), - params.info.clone(), Some(stream_source_core), params.executor_stats.clone(), barrier_receiver, @@ -198,14 +236,18 @@ impl ExecutorBuilder for SourceExecutorBuilder { .boxed() } }; + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); + let rate_limit = source.rate_limit.map(|x| x as _); - Ok(FlowControlExecutor::new(executor, params.actor_context, rate_limit).boxed()) + let exec = + FlowControlExecutor::new((info, exec).into(), params.actor_context, rate_limit); + Ok((params.info, exec).into()) } else { // If there is no external stream source, then no data should be persisted. We pass a // `PanicStateStore` type here for indication. - Ok(SourceExecutor::::new( + let exec = SourceExecutor::::new( params.actor_context, - params.info, None, params.executor_stats, barrier_receiver, @@ -213,8 +255,8 @@ impl ExecutorBuilder for SourceExecutorBuilder { // we don't expect any data in, so no need to set chunk_sizes SourceCtrlOpts::default(), params.env.connector_params(), - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } } diff --git a/src/stream/src/from_proto/stateless_simple_agg.rs b/src/stream/src/from_proto/stateless_simple_agg.rs index c0b5dcb3a4269..2ce5e612737e8 100644 --- a/src/stream/src/from_proto/stateless_simple_agg.rs +++ b/src/stream/src/from_proto/stateless_simple_agg.rs @@ -27,8 +27,7 @@ impl ExecutorBuilder for StatelessSimpleAggExecutorBuilder { params: ExecutorParams, node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let agg_calls: Vec = node .get_agg_calls() @@ -36,9 +35,12 @@ impl ExecutorBuilder for StatelessSimpleAggExecutorBuilder { .map(AggCall::from_protobuf) .try_collect()?; - Ok( - StatelessSimpleAggExecutor::new(params.actor_context, params.info, input, agg_calls)? - .boxed(), - ) + let exec = StatelessSimpleAggExecutor::new( + params.actor_context, + input, + params.info.schema.clone(), + agg_calls, + )?; + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/stream_cdc_scan.rs b/src/stream/src/from_proto/stream_cdc_scan.rs index 27ca849986fcc..2736fdd712cb7 100644 --- a/src/stream/src/from_proto/stream_cdc_scan.rs +++ b/src/stream/src/from_proto/stream_cdc_scan.rs @@ -23,7 +23,7 @@ use risingwave_pb::stream_plan::StreamCdcScanNode; use super::*; use crate::common::table::state_table::StateTable; -use crate::executor::{CdcBackfillExecutor, ExternalStorageTable}; +use crate::executor::{CdcBackfillExecutor, Executor, ExternalStorageTable, FlowControlExecutor}; pub struct StreamCdcScanExecutorBuilder; @@ -34,8 +34,7 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [upstream]: [_; 1] = params.input.try_into().unwrap(); let output_indices = node @@ -45,6 +44,7 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder { .collect_vec(); let table_desc: &ExternalTableDesc = node.get_cdc_table_desc()?; + let disable_backfill = node.disable_backfill; let table_schema: Schema = table_desc.columns.iter().map(Into::into).collect(); assert_eq!(output_indices, (0..table_schema.len()).collect_vec()); @@ -88,18 +88,32 @@ impl ExecutorBuilder for StreamCdcScanExecutorBuilder { let state_table = StateTable::from_table_catalog(node.get_state_table()?, state_store, vnodes).await; - // TODO(kwannoel): Should we apply flow control here as well? - Ok(CdcBackfillExecutor::new( + // adjust backfill chunk size if rate limit is set. + let chunk_size = params.env.config().developer.chunk_size; + let backfill_chunk_size = node + .rate_limit + .map(|x| std::cmp::min(x as usize, chunk_size)) + .unwrap_or(chunk_size); + + let exec = CdcBackfillExecutor::new( params.actor_context.clone(), - params.info, external_table, upstream, output_indices, None, params.executor_stats, state_table, - params.env.config().developer.chunk_size, - ) - .boxed()) + backfill_chunk_size, + disable_backfill, + ); + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); + + let exec = FlowControlExecutor::new( + (info, exec).into(), + params.actor_context, + node.rate_limit.map(|x| x as _), + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/stream_scan.rs b/src/stream/src/from_proto/stream_scan.rs index a8868725d1925..5f0ae484e04c8 100644 --- a/src/stream/src/from_proto/stream_scan.rs +++ b/src/stream/src/from_proto/stream_scan.rs @@ -37,12 +37,11 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { params: ExecutorParams, node: &Self::Node, state_store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [upstream, snapshot]: [_; 2] = params.input.try_into().unwrap(); // For reporting the progress. - let progress = stream - .context + let progress = params + .local_barrier_manager .register_create_mview_progress(params.actor_context.id); let output_indices = node @@ -51,13 +50,13 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { .map(|&i| i as usize) .collect_vec(); - let executor = match node.stream_scan_type() { + let exec = match node.stream_scan_type() { StreamScanType::Chain | StreamScanType::UpstreamOnly => { let upstream_only = matches!(node.stream_scan_type(), StreamScanType::UpstreamOnly); - ChainExecutor::new(params.info, snapshot, upstream, progress, upstream_only).boxed() + ChainExecutor::new(snapshot, upstream, progress, upstream_only).boxed() } StreamScanType::Rearrange => { - RearrangedChainExecutor::new(params.info, snapshot, upstream, progress).boxed() + RearrangedChainExecutor::new(snapshot, upstream, progress).boxed() } StreamScanType::Backfill => { @@ -84,13 +83,12 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { StorageTable::new_partial(state_store.clone(), column_ids, vnodes, table_desc); BackfillExecutor::new( - params.info, upstream_table, upstream, state_table, output_indices, progress, - stream.streaming_metrics.clone(), + params.executor_stats.clone(), params.env.config().developer.chunk_size, node.rate_limit.map(|x| x as _), ) @@ -127,13 +125,12 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { ) .await; ArrangementBackfillExecutor::<_, $SD>::new( - params.info, upstream_table, upstream, state_table, output_indices, progress, - stream.streaming_metrics.clone(), + params.executor_stats.clone(), params.env.config().developer.chunk_size, node.rate_limit.map(|x| x as _), ) @@ -148,11 +145,14 @@ impl ExecutorBuilder for StreamScanExecutorBuilder { } StreamScanType::Unspecified => unreachable!(), }; - Ok(FlowControlExecutor::new( - executor, + let mut info = params.info.clone(); + info.identity = format!("{} (flow controlled)", info.identity); + + let exec = FlowControlExecutor::new( + (info, exec).into(), params.actor_context, node.rate_limit.map(|x| x as _), - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/subscription.rs b/src/stream/src/from_proto/subscription.rs new file mode 100644 index 0000000000000..abc30a1967435 --- /dev/null +++ b/src/stream/src/from_proto/subscription.rs @@ -0,0 +1,81 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use risingwave_common::catalog::{TableId, TableOption}; +use risingwave_pb::stream_plan::SubscriptionNode; +use risingwave_storage::store::{NewLocalOptions, OpConsistencyLevel}; + +use super::ExecutorBuilder; +use crate::common::log_store_impl::kv_log_store::serde::LogStoreRowSerde; +use crate::common::log_store_impl::kv_log_store::KV_LOG_STORE_V2_INFO; +use crate::common::log_store_impl::subscription_log_store::SubscriptionLogStoreWriter; +use crate::error::StreamResult; +use crate::executor::{Executor, SubscriptionExecutor}; + +pub struct SubscriptionExecutorBuilder; + +impl ExecutorBuilder for SubscriptionExecutorBuilder { + type Node = SubscriptionNode; + + async fn new_boxed_executor( + params: crate::task::ExecutorParams, + node: &Self::Node, + state_store: impl risingwave_storage::StateStore, + ) -> StreamResult { + let [input]: [_; 1] = params.input.try_into().unwrap(); + let table_id = TableId::new(node.log_store_table.as_ref().unwrap().id); + let local_state_store = state_store + .new_local(NewLocalOptions { + table_id: TableId { + table_id: node.log_store_table.as_ref().unwrap().id, + }, + op_consistency_level: OpConsistencyLevel::Inconsistent, + table_option: TableOption { + retention_seconds: None, + }, + is_replicated: false, + }) + .await; + + let vnodes = std::sync::Arc::new( + params + .vnode_bitmap + .expect("vnodes not set for subscription"), + ); + let serde = LogStoreRowSerde::new( + node.log_store_table.as_ref().unwrap(), + Some(vnodes.clone()), + &KV_LOG_STORE_V2_INFO, + ); + let log_store_identity = format!( + "subscription[{}]-executor[{}]", + node.subscription_catalog.as_ref().unwrap().id, + params.executor_id + ); + let log_store = + SubscriptionLogStoreWriter::new(table_id, local_state_store, serde, log_store_identity); + let exec = SubscriptionExecutor::new( + params.actor_context, + input, + log_store, + node.subscription_catalog + .as_ref() + .unwrap() + .properties + .clone(), + ) + .await?; + Ok((params.info, exec).into()) + } +} diff --git a/src/stream/src/from_proto/temporal_join.rs b/src/stream/src/from_proto/temporal_join.rs index f57fbbdac1f1c..15badec97e5cc 100644 --- a/src/stream/src/from_proto/temporal_join.rs +++ b/src/stream/src/from_proto/temporal_join.rs @@ -35,8 +35,7 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let table_desc: &StorageTableDesc = node.get_table_desc()?; let table = { let column_ids = table_desc @@ -102,7 +101,7 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder { let dispatcher_args = TemporalJoinExecutorDispatcherArgs { ctx: params.actor_context, - info: params.info, + info: params.info.clone(), left: source_l, right: source_r, right_table: table, @@ -113,22 +112,22 @@ impl ExecutorBuilder for TemporalJoinExecutorBuilder { output_indices, table_output_indices, table_stream_key_indices, - watermark_epoch: stream.get_watermark_epoch(), + watermark_epoch: params.watermark_epoch, chunk_size: params.env.config().developer.chunk_size, metrics: params.executor_stats, join_type_proto: node.get_join_type()?, join_key_data_types, }; - dispatcher_args.dispatch() + Ok((params.info, dispatcher_args.dispatch()?).into()) } } struct TemporalJoinExecutorDispatcherArgs { ctx: ActorContextRef, info: ExecutorInfo, - left: BoxedExecutor, - right: BoxedExecutor, + left: Executor, + right: Executor, right_table: StorageTable, left_join_keys: Vec, right_join_keys: Vec, @@ -145,7 +144,7 @@ struct TemporalJoinExecutorDispatcherArgs { } impl HashKeyDispatcher for TemporalJoinExecutorDispatcherArgs { - type Output = StreamResult; + type Output = StreamResult>; fn dispatch_impl(self) -> Self::Output { /// This macro helps to fill the const generic type parameter. diff --git a/src/stream/src/from_proto/top_n.rs b/src/stream/src/from_proto/top_n.rs index a4bcf51277cfa..10071534e57df 100644 --- a/src/stream/src/from_proto/top_n.rs +++ b/src/stream/src/from_proto/top_n.rs @@ -30,8 +30,7 @@ impl ExecutorBuilder for TopNExecutorBuilder StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let table = node.get_table()?; @@ -53,7 +52,7 @@ impl ExecutorBuilder for TopNExecutorBuilder::new( input, params.actor_context, - params.info, + params.info.schema.clone(), storage_key, (node.offset as usize, node.limit as usize), order_by, @@ -63,11 +62,12 @@ impl ExecutorBuilder for TopNExecutorBuilder> = match (APPEND_ONLY, node.with_ties) { (true, true) => build!(AppendOnlyTopNExecutor, true), (true, false) => build!(AppendOnlyTopNExecutor, false), (false, true) => build!(TopNExecutor, true), (false, false) => build!(TopNExecutor, false), - } + }; + Ok((params.info, exec?).into()) } } diff --git a/src/stream/src/from_proto/union.rs b/src/stream/src/from_proto/union.rs index 485c0bd0a35e3..1360f92c1c83b 100644 --- a/src/stream/src/from_proto/union.rs +++ b/src/stream/src/from_proto/union.rs @@ -26,8 +26,7 @@ impl ExecutorBuilder for UnionExecutorBuilder { params: ExecutorParams, _node: &Self::Node, _store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { - Ok(UnionExecutor::new(params.info, params.input).boxed()) + ) -> StreamResult { + Ok((params.info, UnionExecutor::new(params.input)).into()) } } diff --git a/src/stream/src/from_proto/values.rs b/src/stream/src/from_proto/values.rs index a45393e200a5e..28e9ca086414b 100644 --- a/src/stream/src/from_proto/values.rs +++ b/src/stream/src/from_proto/values.rs @@ -20,8 +20,8 @@ use tokio::sync::mpsc::unbounded_channel; use super::ExecutorBuilder; use crate::error::StreamResult; -use crate::executor::{BoxedExecutor, ValuesExecutor}; -use crate::task::{ExecutorParams, LocalStreamManagerCore}; +use crate::executor::{Executor, ValuesExecutor}; +use crate::task::ExecutorParams; /// Build a `ValuesExecutor` for stream. As is a leaf, current workaround registers a `sender` for /// this executor. May refractor with `BarrierRecvExecutor` in the near future. @@ -34,15 +34,13 @@ impl ExecutorBuilder for ValuesExecutorBuilder { params: ExecutorParams, node: &ValuesNode, _store: impl StateStore, - stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let (sender, barrier_receiver) = unbounded_channel(); - stream - .context - .barrier_manager() + params + .local_barrier_manager .register_sender(params.actor_context.id, sender); - let progress = stream - .context + let progress = params + .local_barrier_manager .register_create_mview_progress(params.actor_context.id); let rows = node .get_tuples() @@ -57,12 +55,13 @@ impl ExecutorBuilder for ValuesExecutorBuilder { .collect_vec() }) .collect_vec(); - Ok(Box::new(ValuesExecutor::new( + let exec = ValuesExecutor::new( params.actor_context, - params.info, + params.info.schema.clone(), progress, rows, barrier_receiver, - ))) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/from_proto/watermark_filter.rs b/src/stream/src/from_proto/watermark_filter.rs index 44618e812fd82..f01695e991487 100644 --- a/src/stream/src/from_proto/watermark_filter.rs +++ b/src/stream/src/from_proto/watermark_filter.rs @@ -34,8 +34,7 @@ impl ExecutorBuilder for WatermarkFilterBuilder { params: ExecutorParams, node: &Self::Node, store: impl StateStore, - _stream: &mut LocalStreamManagerCore, - ) -> StreamResult { + ) -> StreamResult { let [input]: [_; 1] = params.input.try_into().unwrap(); let watermark_descs = node.get_watermark_descs().clone(); let [watermark_desc]: [_; 1] = watermark_descs.try_into().unwrap(); @@ -64,15 +63,15 @@ impl ExecutorBuilder for WatermarkFilterBuilder { let table = StateTable::from_table_catalog_inconsistent_op(&table, store, Some(vnodes)).await; - Ok(WatermarkFilterExecutor::new( + let exec = WatermarkFilterExecutor::new( params.actor_context, - params.info, + ¶ms.info, input, watermark_expr, event_time_col_idx, table, global_watermark_table, - ) - .boxed()) + ); + Ok((params.info, exec).into()) } } diff --git a/src/stream/src/lib.rs b/src/stream/src/lib.rs index 4c452d4bb132b..7b036ed520a1d 100644 --- a/src/stream/src/lib.rs +++ b/src/stream/src/lib.rs @@ -40,6 +40,7 @@ #![feature(is_sorted)] #![feature(btree_cursors)] #![feature(assert_matches)] +#![feature(try_blocks)] #[macro_use] extern crate tracing; diff --git a/src/stream/src/task/barrier_manager.rs b/src/stream/src/task/barrier_manager.rs index b706d7a5c3537..edbd660690049 100644 --- a/src/stream/src/task/barrier_manager.rs +++ b/src/stream/src/task/barrier_manager.rs @@ -14,17 +14,25 @@ use std::collections::{HashMap, HashSet}; use std::sync::Arc; +use std::time::Duration; -use anyhow::anyhow; +use anyhow::{anyhow, Context}; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use parking_lot::Mutex; use risingwave_pb::stream_service::barrier_complete_response::PbCreateMviewProgress; +use rw_futures_util::{pending_on_none, AttachedFuture}; use thiserror_ext::AsReport; +use tokio::select; use tokio::sync::mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender}; use tokio::sync::oneshot; -use tokio::sync::oneshot::Receiver; +use tokio::task::JoinHandle; use self::managed_state::ManagedBarrierState; use crate::error::{IntoUnexpectedExit, StreamError, StreamResult}; -use crate::task::ActorId; +use crate::task::{ + ActorHandle, ActorId, AtomicU64Ref, SharedContext, StreamEnvironment, UpDownActorIds, +}; mod managed_state; mod progress; @@ -32,13 +40,16 @@ mod progress; mod tests; pub use progress::CreateMviewProgress; +use risingwave_common::util::runtime::BackgroundShutdownRuntime; +use risingwave_pb::common::ActorInfo; +use risingwave_pb::stream_plan; use risingwave_pb::stream_plan::barrier::BarrierKind; -use risingwave_storage::StateStoreImpl; +use risingwave_storage::store::SyncResult; +use crate::executor::exchange::permit::Receiver; use crate::executor::monitor::StreamingMetrics; -use crate::executor::Barrier; +use crate::executor::{Actor, Barrier, DispatchExecutor}; use crate::task::barrier_manager::progress::BackfillState; -use crate::task::barrier_manager::LocalBarrierEvent::{ReportActorCollected, ReportActorFailure}; /// If enabled, all actors will be grouped in the same tracing span within one epoch. /// Note that this option will significantly increase the overhead of tracing. @@ -46,38 +57,23 @@ pub const ENABLE_BARRIER_AGGREGATION: bool = false; /// Collect result of some barrier on current compute node. Will be reported to the meta service. #[derive(Debug)] -pub struct CollectResult { +pub struct BarrierCompleteResult { + /// The result returned from `sync` of `StateStore`. + pub sync_result: Option, + /// The updated creation progress of materialized view after this barrier. pub create_mview_progress: Vec, - - /// The kind of barrier. - pub kind: BarrierKind, } -enum LocalBarrierEvent { +pub(super) enum LocalBarrierEvent { RegisterSender { actor_id: ActorId, sender: UnboundedSender, }, - InjectBarrier { - barrier: Barrier, - actor_ids_to_send: HashSet, - actor_ids_to_collect: HashSet, - result_sender: oneshot::Sender>, - }, - Reset, ReportActorCollected { actor_id: ActorId, barrier: Barrier, }, - ReportActorFailure { - actor_id: ActorId, - err: StreamError, - }, - CollectEpoch { - epoch: u64, - result_sender: oneshot::Sender>, - }, ReportCreateProgress { current_epoch: u64, actor: ActorId, @@ -87,84 +83,287 @@ enum LocalBarrierEvent { Flush(oneshot::Sender<()>), } +pub(super) enum LocalActorOperation { + InjectBarrier { + barrier: Barrier, + actor_ids_to_send: HashSet, + actor_ids_to_collect: HashSet, + result_sender: oneshot::Sender>, + }, + Reset { + prev_epoch: u64, + result_sender: oneshot::Sender<()>, + }, + AwaitEpochCompleted { + epoch: u64, + result_sender: oneshot::Sender>, + }, + DropActors { + actors: Vec, + result_sender: oneshot::Sender<()>, + }, + UpdateActors { + actors: Vec, + result_sender: oneshot::Sender>, + }, + BuildActors { + actors: Vec, + result_sender: oneshot::Sender>, + }, + UpdateActorInfo { + new_actor_infos: Vec, + result_sender: oneshot::Sender>, + }, + TakeReceiver { + ids: UpDownActorIds, + result_sender: oneshot::Sender>, + }, + #[cfg(test)] + GetCurrentSharedContext(oneshot::Sender>), +} + +pub(crate) struct StreamActorManagerState { + /// Each processor runs in a future. Upon receiving a `Terminate` message, they will exit. + /// `handles` store join handles of these futures, and therefore we could wait their + /// termination. + pub(super) handles: HashMap, + + /// Stores all actor information, taken after actor built. + pub(super) actors: HashMap, + + /// Stores all actor tokio runtime monitoring tasks. + pub(super) actor_monitor_tasks: HashMap, + + #[expect(clippy::type_complexity)] + pub(super) creating_actors: FuturesUnordered< + AttachedFuture< + JoinHandle>>>, + oneshot::Sender>, + >, + >, +} + +impl StreamActorManagerState { + fn new() -> Self { + Self { + handles: HashMap::new(), + actors: HashMap::new(), + actor_monitor_tasks: HashMap::new(), + creating_actors: FuturesUnordered::new(), + } + } + + async fn next_created_actors( + &mut self, + ) -> ( + oneshot::Sender>, + StreamResult>>, + ) { + let (join_result, sender) = pending_on_none(self.creating_actors.next()).await; + ( + sender, + try { join_result.context("failed to join creating actors futures")?? }, + ) + } +} + +pub(crate) struct StreamActorManager { + pub(super) env: StreamEnvironment, + pub(super) streaming_metrics: Arc, + + /// Watermark epoch number. + pub(super) watermark_epoch: AtomicU64Ref, + + /// Manages the await-trees of all actors. + pub(super) await_tree_reg: Option>>>, + + /// Runtime for the streaming actors. + pub(super) runtime: BackgroundShutdownRuntime, +} + /// [`LocalBarrierWorker`] manages barrier control flow, used by local stream manager. /// Specifically, [`LocalBarrierWorker`] serve barrier injection from meta server, send the /// barriers to and collect them from all actors, and finally report the progress. -struct LocalBarrierWorker { +pub(super) struct LocalBarrierWorker { /// Stores all streaming job source sender. barrier_senders: HashMap>>, /// Current barrier collection state. state: ManagedBarrierState, - /// Save collect `CompleteReceiver`. - collect_complete_receiver: HashMap, - /// Record all unexpected exited actors. failure_actors: HashMap, -} -/// Information used after collection. -pub struct CompleteReceiver { - /// Notify all actors of completion of collection. - pub complete_receiver: Option>>, - /// The kind of barrier. - pub kind: BarrierKind, + epoch_result_sender: HashMap>>, + + pub(super) actor_manager: Arc, + + pub(super) actor_manager_state: StreamActorManagerState, + + pub(super) current_shared_context: Arc, + + barrier_event_rx: UnboundedReceiver, + + actor_failure_rx: UnboundedReceiver<(ActorId, StreamError)>, + + root_failure: Option, } impl LocalBarrierWorker { - fn new(state_store: StateStoreImpl, streaming_metrics: Arc) -> Self { + pub(super) fn new(actor_manager: Arc) -> Self { + let (event_tx, event_rx) = unbounded_channel(); + let (failure_tx, failure_rx) = unbounded_channel(); + let shared_context = Arc::new(SharedContext::new( + actor_manager.env.server_address().clone(), + actor_manager.env.config(), + LocalBarrierManager { + barrier_event_sender: event_tx, + actor_failure_sender: failure_tx, + }, + )); Self { barrier_senders: HashMap::new(), failure_actors: HashMap::default(), - state: ManagedBarrierState::new(state_store, streaming_metrics), - collect_complete_receiver: HashMap::default(), + state: ManagedBarrierState::new( + actor_manager.env.state_store(), + actor_manager.streaming_metrics.clone(), + ), + epoch_result_sender: HashMap::default(), + actor_manager, + actor_manager_state: StreamActorManagerState::new(), + current_shared_context: shared_context, + barrier_event_rx: event_rx, + actor_failure_rx: failure_rx, + root_failure: None, } } - async fn run(mut self, mut event_rx: UnboundedReceiver) { - while let Some(event) = event_rx.recv().await { - match event { - LocalBarrierEvent::RegisterSender { actor_id, sender } => { - self.register_sender(actor_id, sender); - } - LocalBarrierEvent::InjectBarrier { - barrier, - actor_ids_to_send, - actor_ids_to_collect, - result_sender, - } => { - let result = - self.send_barrier(&barrier, actor_ids_to_send, actor_ids_to_collect); - let _ = result_sender.send(result).inspect_err(|e| { - warn!(err=?e, "fail to send inject barrier result"); - }); - } - LocalBarrierEvent::Reset => { - self.reset(); - } - ReportActorCollected { actor_id, barrier } => self.collect(actor_id, &barrier), - ReportActorFailure { actor_id, err } => { - self.notify_failure(actor_id, err); + async fn run(mut self, mut actor_op_rx: UnboundedReceiver) { + loop { + select! { + biased; + (sender, create_actors_result) = self.actor_manager_state.next_created_actors() => { + self.handle_actor_created(sender, create_actors_result); } - LocalBarrierEvent::CollectEpoch { - epoch, - result_sender, - } => { - let result = self.remove_collect_rx(epoch); - let _ = result_sender.send(result).inspect_err(|e| { - warn!(err=?e.as_ref().map(|_|()), "fail to send collect epoch result"); - }); - } - LocalBarrierEvent::ReportCreateProgress { - current_epoch, - actor, - state, - } => { - self.update_create_mview_progress(current_epoch, actor, state); + completed_epoch = self.state.next_completed_epoch() => { + self.on_epoch_completed(completed_epoch); + }, + // Note: it's important to select in a biased way to ensure that + // barrier event is handled before actor_op, because we must ensure + // that register sender is handled before inject barrier. + event = self.barrier_event_rx.recv() => { + self.handle_barrier_event(event.expect("should not be none")); + }, + failure = self.actor_failure_rx.recv() => { + let (actor_id, err) = failure.unwrap(); + self.notify_failure(actor_id, err).await; + }, + actor_op = actor_op_rx.recv() => { + if let Some(actor_op) = actor_op { + match actor_op { + LocalActorOperation::Reset { + result_sender, prev_epoch} => { + self.reset(prev_epoch).await; + let _ = result_sender.send(()); + } + actor_op => { + self.handle_actor_op(actor_op); + } + } + } + else { + break; + } } - #[cfg(test)] - LocalBarrierEvent::Flush(sender) => sender.send(()).unwrap(), + } + } + } + + fn handle_actor_created( + &mut self, + sender: oneshot::Sender>, + create_actor_result: StreamResult>>, + ) { + let result = create_actor_result.map(|actors| { + self.spawn_actors(actors); + }); + + let _ = sender.send(result); + } + + fn handle_barrier_event(&mut self, event: LocalBarrierEvent) { + match event { + LocalBarrierEvent::RegisterSender { actor_id, sender } => { + self.register_sender(actor_id, sender); + } + LocalBarrierEvent::ReportActorCollected { actor_id, barrier } => { + self.collect(actor_id, &barrier) + } + LocalBarrierEvent::ReportCreateProgress { + current_epoch, + actor, + state, + } => { + self.update_create_mview_progress(current_epoch, actor, state); + } + #[cfg(test)] + LocalBarrierEvent::Flush(sender) => sender.send(()).unwrap(), + } + } + + fn handle_actor_op(&mut self, actor_op: LocalActorOperation) { + match actor_op { + LocalActorOperation::InjectBarrier { + barrier, + actor_ids_to_send, + actor_ids_to_collect, + result_sender, + } => { + let result = self.send_barrier(&barrier, actor_ids_to_send, actor_ids_to_collect); + let _ = result_sender.send(result).inspect_err(|e| { + warn!(err=?e, "fail to send inject barrier result"); + }); + } + LocalActorOperation::Reset { .. } => { + unreachable!("Reset event should be handled separately in async context") + } + + LocalActorOperation::AwaitEpochCompleted { + epoch, + result_sender, + } => { + self.await_epoch_completed(epoch, result_sender); + } + LocalActorOperation::DropActors { + actors, + result_sender, + } => { + self.drop_actors(&actors); + let _ = result_sender.send(()); + } + LocalActorOperation::UpdateActors { + actors, + result_sender, + } => { + let result = self.update_actors(actors); + let _ = result_sender.send(result); + } + LocalActorOperation::BuildActors { + actors, + result_sender, + } => self.start_create_actors(&actors, result_sender), + LocalActorOperation::UpdateActorInfo { + new_actor_infos, + result_sender, + } => { + let _ = result_sender.send(self.update_actor_info(new_actor_infos)); + } + LocalActorOperation::TakeReceiver { ids, result_sender } => { + let _ = result_sender.send(self.current_shared_context.take_receiver(ids)); + } + #[cfg(test)] + LocalActorOperation::GetCurrentSharedContext(sender) => { + let _ = sender.send(self.current_shared_context.clone()); } } } @@ -172,6 +371,19 @@ impl LocalBarrierWorker { // event handler impl LocalBarrierWorker { + fn on_epoch_completed(&mut self, epoch: u64) { + if let Some(sender) = self.epoch_result_sender.remove(&epoch) { + let result = self + .state + .pop_completed_epoch(epoch) + .expect("should exist") + .expect("should have completed"); + if sender.send(result).is_err() { + warn!(epoch, "fail to send epoch complete result"); + } + } + } + /// Register sender for source actors, used to send barriers. fn register_sender(&mut self, actor_id: ActorId, sender: UnboundedSender) { tracing::debug!( @@ -193,6 +405,32 @@ impl LocalBarrierWorker { to_send: HashSet, to_collect: HashSet, ) -> StreamResult<()> { + #[cfg(not(test))] + { + use itertools::Itertools; + // The barrier might be outdated and been injected after recovery in some certain extreme + // scenarios. So some newly creating actors in the barrier are possibly not rebuilt during + // recovery. Check it here and return an error here if some actors are not found to + // avoid collection hang. We need some refine in meta side to remove this workaround since + // it will cause another round of unnecessary recovery. + let missing_actor_ids = to_collect + .iter() + .filter(|id| !self.actor_manager_state.handles.contains_key(id)) + .collect_vec(); + if !missing_actor_ids.is_empty() { + tracing::warn!( + "to collect actors not found, they should be cleaned when recovering: {:?}", + missing_actor_ids + ); + return Err(anyhow!("to collect actors not found: {:?}", to_collect).into()); + } + } + + if barrier.kind == BarrierKind::Initial { + self.actor_manager + .watermark_epoch + .store(barrier.epoch.curr, std::sync::atomic::Ordering::SeqCst); + } debug!( target: "events::stream::barrier::manager::send", "send barrier {:?}, senders = {:?}, actor_ids_to_collect = {:?}", @@ -209,12 +447,12 @@ impl LocalBarrierWorker { // The failure actors could exit before the barrier is issued, while their // up-downstream actors could be stuck somehow. Return error directly to trigger the // recovery. - return Err(e.clone()); + // try_find_root_failure is not used merely because it requires async. + return Err(self.root_failure.clone().unwrap_or(e.clone())); } } - let (tx, rx) = oneshot::channel(); - self.state.transform_to_issued(barrier, to_collect, tx); + self.state.transform_to_issued(barrier, to_collect); for actor_id in to_send { match self.barrier_senders.get(&actor_id) { @@ -251,39 +489,42 @@ impl LocalBarrierWorker { self.barrier_senders.remove(actor); } } - - self.collect_complete_receiver.insert( - barrier.epoch.prev, - CompleteReceiver { - complete_receiver: Some(rx), - kind: barrier.kind, - }, - ); Ok(()) } /// Use `prev_epoch` to remove collect rx and return rx. - fn remove_collect_rx(&mut self, prev_epoch: u64) -> StreamResult { - // It's still possible that `collect_complete_receiver` does not contain the target epoch - // when receiving collect_barrier request. Because `collect_complete_receiver` could - // be cleared when CN is under recovering. We should return error rather than panic. - self.collect_complete_receiver - .remove(&prev_epoch) - .ok_or_else(|| { - anyhow!( - "barrier collect complete receiver for prev epoch {} not exists", - prev_epoch - ) - .into() - }) + fn await_epoch_completed( + &mut self, + prev_epoch: u64, + result_sender: oneshot::Sender>, + ) { + match self.state.pop_completed_epoch(prev_epoch) { + Err(e) => { + let _ = result_sender.send(Err(e)); + } + Ok(Some(result)) => { + if result_sender.send(result).is_err() { + warn!(prev_epoch, "failed to send completed epoch result"); + } + } + Ok(None) => { + if let Some(prev_sender) = + self.epoch_result_sender.insert(prev_epoch, result_sender) + { + warn!(?prev_epoch, "duplicate await_collect_barrier on epoch"); + let _ = prev_sender.send(Err(anyhow!( + "duplicate await_collect_barrier on epoch {}", + prev_epoch + ) + .into())); + } + } + } } /// Reset all internal states. - fn reset(&mut self) { - *self = Self::new( - self.state.state_store.clone(), - self.state.streaming_metrics.clone(), - ); + pub(super) fn reset_state(&mut self) { + *self = Self::new(self.actor_manager.clone()); } /// When a [`crate::executor::StreamConsumer`] (typically [`crate::executor::DispatchExecutor`]) get a barrier, it should report @@ -294,96 +535,157 @@ impl LocalBarrierWorker { /// When a actor exit unexpectedly, it should report this event using this function, so meta /// will notice actor's exit while collecting. - fn notify_failure(&mut self, actor_id: ActorId, err: StreamError) { + async fn notify_failure(&mut self, actor_id: ActorId, err: StreamError) { + self.add_failure(actor_id, err.clone()); + let root_err = self.try_find_root_failure(err).await; + for fail_epoch in self.state.epochs_await_on_actor(actor_id) { + if let Some(result_sender) = self.epoch_result_sender.remove(&fail_epoch) { + if result_sender.send(Err(root_err.clone())).is_err() { + warn!(fail_epoch, actor_id, err = %root_err.as_report(), "fail to notify actor failure"); + } + } + } + } + + fn add_failure(&mut self, actor_id: ActorId, err: StreamError) { let err = err.into_unexpected_exit(actor_id); - if let Some(prev_err) = self.failure_actors.insert(actor_id, err.clone()) { + if let Some(prev_err) = self.failure_actors.insert(actor_id, err) { warn!( actor_id, prev_err = %prev_err.as_report(), "actor error overwritten" ); } - for (fail_epoch, notifier) in self.state.notifiers_await_on_actor(actor_id) { - if notifier.send(Err(err.clone())).is_err() { - warn!( - fail_epoch, - actor_id, - err = %err.as_report(), - "fail to notify actor failure" - ); - } + } + + async fn try_find_root_failure(&mut self, default_err: StreamError) -> StreamError { + if let Some(root_failure) = &self.root_failure { + return root_failure.clone(); } + // fetch more actor errors within a timeout + let _ = tokio::time::timeout(Duration::from_secs(3), async { + while let Some((actor_id, error)) = self.actor_failure_rx.recv().await { + self.add_failure(actor_id, error); + } + }) + .await; + self.root_failure = try_find_root_actor_failure(self.failure_actors.values()); + self.root_failure.clone().unwrap_or(default_err) } } #[derive(Clone)] pub struct LocalBarrierManager { barrier_event_sender: UnboundedSender, + actor_failure_sender: UnboundedSender<(ActorId, StreamError)>, } -impl LocalBarrierManager { +impl LocalBarrierWorker { /// Create a [`LocalBarrierWorker`] with managed mode. - pub fn new(state_store: StateStoreImpl, streaming_metrics: Arc) -> Self { - let (tx, rx) = unbounded_channel(); - let worker = LocalBarrierWorker::new(state_store, streaming_metrics); - let _join_handle = tokio::spawn(worker.run(rx)); - Self { - barrier_event_sender: tx, - } + pub fn spawn( + env: StreamEnvironment, + streaming_metrics: Arc, + await_tree_reg: Option>>>, + watermark_epoch: AtomicU64Ref, + actor_op_rx: UnboundedReceiver, + ) -> JoinHandle<()> { + let runtime = { + let mut builder = tokio::runtime::Builder::new_multi_thread(); + if let Some(worker_threads_num) = env.config().actor_runtime_worker_threads_num { + builder.worker_threads(worker_threads_num); + } + builder + .thread_name("rw-streaming") + .enable_all() + .build() + .unwrap() + }; + + let actor_manager = Arc::new(StreamActorManager { + env: env.clone(), + streaming_metrics, + watermark_epoch, + await_tree_reg, + runtime: runtime.into(), + }); + let worker = LocalBarrierWorker::new(actor_manager); + tokio::spawn(worker.run(actor_op_rx)) } +} - fn send_event(&self, event: LocalBarrierEvent) { - self.barrier_event_sender - .send(event) - .expect("should be able to send event") +pub(super) struct EventSender(pub(super) UnboundedSender); + +impl Clone for EventSender { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl EventSender { + pub(super) fn send_event(&self, event: T) { + self.0.send(event).expect("should be able to send event") + } + + pub(super) async fn send_and_await( + &self, + make_event: impl FnOnce(oneshot::Sender) -> T, + ) -> StreamResult { + let (tx, rx) = oneshot::channel(); + let event = make_event(tx); + self.send_event(event); + rx.await + .map_err(|_| anyhow!("barrier manager maybe reset").into()) } } impl LocalBarrierManager { + fn send_event(&self, event: LocalBarrierEvent) { + // ignore error, because the current barrier manager maybe a stale one + let _ = self.barrier_event_sender.send(event); + } + /// Register sender for source actors, used to send barriers. pub fn register_sender(&self, actor_id: ActorId, sender: UnboundedSender) { self.send_event(LocalBarrierEvent::RegisterSender { actor_id, sender }); } +} +impl EventSender { /// Broadcast a barrier to all senders. Save a receiver which will get notified when this /// barrier is finished, in managed mode. - pub async fn send_barrier( + pub(super) async fn send_barrier( &self, barrier: Barrier, actor_ids_to_send: impl IntoIterator, actor_ids_to_collect: impl IntoIterator, ) -> StreamResult<()> { - let (tx, rx) = oneshot::channel(); - self.send_event(LocalBarrierEvent::InjectBarrier { + self.send_and_await(move |result_sender| LocalActorOperation::InjectBarrier { barrier, actor_ids_to_send: actor_ids_to_send.into_iter().collect(), actor_ids_to_collect: actor_ids_to_collect.into_iter().collect(), - result_sender: tx, - }); - rx.await - .map_err(|_| anyhow!("barrier manager maybe reset"))? + result_sender, + }) + .await? } /// Use `prev_epoch` to remove collect rx and return rx. - pub async fn remove_collect_rx(&self, prev_epoch: u64) -> StreamResult { - let (tx, rx) = oneshot::channel(); - self.send_event(LocalBarrierEvent::CollectEpoch { + pub(super) async fn await_epoch_completed( + &self, + prev_epoch: u64, + ) -> StreamResult { + self.send_and_await(|result_sender| LocalActorOperation::AwaitEpochCompleted { epoch: prev_epoch, - result_sender: tx, - }); - rx.await - .map_err(|_| anyhow!("barrier manager maybe reset"))? - } - - /// Reset all internal states. - pub fn reset(&self) { - self.send_event(LocalBarrierEvent::Reset) + result_sender, + }) + .await? } +} +impl LocalBarrierManager { /// When a [`crate::executor::StreamConsumer`] (typically [`crate::executor::DispatchExecutor`]) get a barrier, it should report /// and collect this barrier with its own `actor_id` using this function. pub fn collect(&self, actor_id: ActorId, barrier: &Barrier) { - self.send_event(ReportActorCollected { + self.send_event(LocalBarrierEvent::ReportActorCollected { actor_id, barrier: barrier.clone(), }) @@ -392,17 +694,69 @@ impl LocalBarrierManager { /// When a actor exit unexpectedly, it should report this event using this function, so meta /// will notice actor's exit while collecting. pub fn notify_failure(&self, actor_id: ActorId, err: StreamError) { - self.send_event(ReportActorFailure { actor_id, err }) + let _ = self.actor_failure_sender.send((actor_id, err)); } } +/// Tries to find the root cause of actor failures, based on hard-coded rules. +pub fn try_find_root_actor_failure<'a>( + actor_errors: impl IntoIterator, +) -> Option { + use crate::executor::StreamExecutorError; + let stream_executor_error_score = |e: &StreamExecutorError| { + use crate::executor::error::ErrorKind; + match e.inner() { + ErrorKind::ChannelClosed(_) => 0, + ErrorKind::Internal(_) => 1, + _ => 999, + } + }; + let stream_error_score = |e: &&StreamError| { + use crate::error::ErrorKind; + match e.inner() { + ErrorKind::Internal(_) => 1000, + ErrorKind::Executor(ee) => 2000 + stream_executor_error_score(ee), + _ => 3000, + } + }; + actor_errors + .into_iter() + .max_by_key(stream_error_score) + .cloned() +} + #[cfg(test)] impl LocalBarrierManager { - pub fn for_test() -> Self { - Self::new( - StateStoreImpl::for_test(), + pub(super) async fn spawn_for_test() -> (EventSender, Self) { + use std::sync::atomic::AtomicU64; + let (tx, rx) = unbounded_channel(); + let _join_handle = LocalBarrierWorker::spawn( + StreamEnvironment::for_test(), Arc::new(StreamingMetrics::unused()), - ) + None, + Arc::new(AtomicU64::new(0)), + rx, + ); + let sender = EventSender(tx); + let context = sender + .send_and_await(LocalActorOperation::GetCurrentSharedContext) + .await + .unwrap(); + + (sender, context.local_barrier_manager.clone()) + } + + pub fn for_test() -> Self { + let (tx, mut rx) = unbounded_channel(); + let (failure_tx, failure_rx) = unbounded_channel(); + let _join_handle = tokio::spawn(async move { + let _failure_rx = failure_rx; + while rx.recv().await.is_some() {} + }); + Self { + barrier_event_sender: tx, + actor_failure_sender: failure_tx, + } } pub async fn flush_all_events(&self) { diff --git a/src/stream/src/task/barrier_manager/managed_state.rs b/src/stream/src/task/barrier_manager/managed_state.rs index ef4cc5ece37b2..3970008d948c0 100644 --- a/src/stream/src/task/barrier_manager/managed_state.rs +++ b/src/stream/src/task/barrier_manager/managed_state.rs @@ -12,19 +12,29 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::assert_matches::assert_matches; use std::collections::{BTreeMap, HashMap, HashSet}; +use std::future::Future; use std::iter::once; +use std::mem::replace; use std::ops::Sub; use std::sync::Arc; +use anyhow::anyhow; +use await_tree::InstrumentAwait; +use futures::stream::FuturesOrdered; +use futures::{FutureExt, StreamExt}; use prometheus::HistogramTimer; +use risingwave_common::must_match; use risingwave_pb::stream_plan::barrier::BarrierKind; use risingwave_pb::stream_service::barrier_complete_response::CreateMviewProgress; +use risingwave_storage::store::SyncResult; use risingwave_storage::{dispatch_state_store, StateStore, StateStoreImpl}; -use tokio::sync::oneshot; +use rw_futures_util::pending_on_none; +use thiserror_ext::AsReport; use super::progress::BackfillState; -use super::CollectResult; +use super::BarrierCompleteResult; use crate::error::StreamResult; use crate::executor::monitor::StreamingMetrics; use crate::executor::Barrier; @@ -45,11 +55,15 @@ enum ManagedBarrierStateInner { /// Actor ids remaining to be collected. remaining_actors: HashSet, - /// Notify that the collection is finished. - collect_notifier: Option>>, - barrier_inflight_latency: HistogramTimer, }, + + /// The barrier has been collected by all remaining actors + AllCollected, + + /// The barrier has been completed, which means the barrier has been collected by all actors and + /// synced in state store + Completed(StreamResult), } #[derive(Debug)] @@ -59,6 +73,56 @@ pub(super) struct BarrierState { kind: BarrierKind, } +type AwaitEpochCompletedFuture = + impl Future)> + 'static; + +fn sync_epoch( + state_store: &StateStoreImpl, + streaming_metrics: &StreamingMetrics, + prev_epoch: u64, + kind: BarrierKind, +) -> impl Future>> + 'static { + let barrier_sync_latency = streaming_metrics.barrier_sync_latency.clone(); + let state_store = state_store.clone(); + + async move { + let sync_result = match kind { + BarrierKind::Unspecified => unreachable!(), + BarrierKind::Initial => { + if let Some(hummock) = state_store.as_hummock() { + let mce = hummock.get_pinned_version().max_committed_epoch(); + assert_eq!( + mce, prev_epoch, + "first epoch should match with the current version", + ); + } + tracing::info!(?prev_epoch, "ignored syncing data for the first barrier"); + None + } + BarrierKind::Barrier => None, + BarrierKind::Checkpoint => { + let timer = barrier_sync_latency.start_timer(); + let sync_result = dispatch_state_store!(state_store, store, { + store + .sync(prev_epoch) + .instrument_await(format!("sync_epoch (epoch {})", prev_epoch)) + .await + .inspect_err(|e| { + tracing::error!( + prev_epoch, + error = %e.as_report(), + "Failed to sync state store", + ); + }) + })?; + timer.observe_duration(); + Some(sync_result) + } + }; + Ok(sync_result) + } +} + pub(super) struct ManagedBarrierState { /// Record barrier state for each epoch of concurrent checkpoints. /// @@ -71,6 +135,9 @@ pub(super) struct ManagedBarrierState { pub(super) state_store: StateStoreImpl, pub(super) streaming_metrics: Arc, + + /// Futures will be finished in the order of epoch in ascending order. + await_epoch_completed_futures: FuturesOrdered, } impl ManagedBarrierState { @@ -92,41 +159,45 @@ impl ManagedBarrierState { create_mview_progress: Default::default(), state_store, streaming_metrics, + await_epoch_completed_futures: FuturesOrdered::new(), } } - /// Notify if we have collected barriers from all actor ids. The state must be `Issued`. - fn may_notify(&mut self, prev_epoch: u64) { + /// This method is called when barrier state is modified in either `Issued` or `Stashed` + /// to transform the state to `AllCollected` and start state store `sync` when the barrier + /// has been collected from all actors for an `Issued` barrier. + fn may_have_collected_all(&mut self, prev_epoch: u64) { // Report if there's progress on the earliest in-flight barrier. if self.epoch_barrier_state_map.keys().next() == Some(&prev_epoch) { self.streaming_metrics.barrier_manager_progress.inc(); } - while let Some(entry) = self.epoch_barrier_state_map.first_entry() { - let to_notify = matches!( - &entry.get().inner, + for (prev_epoch, barrier_state) in &mut self.epoch_barrier_state_map { + let prev_epoch = *prev_epoch; + match &barrier_state.inner { ManagedBarrierStateInner::Issued { remaining_actors, .. - } if remaining_actors.is_empty(), - ); - - if !to_notify { - break; + } if remaining_actors.is_empty() => {} + ManagedBarrierStateInner::AllCollected | ManagedBarrierStateInner::Completed(_) => { + continue; + } + ManagedBarrierStateInner::Stashed { .. } + | ManagedBarrierStateInner::Issued { .. } => { + break; + } } - let (prev_epoch, barrier_state) = entry.remove_entry(); + let prev_state = replace( + &mut barrier_state.inner, + ManagedBarrierStateInner::AllCollected, + ); - let collect_notifier = match barrier_state.inner { - ManagedBarrierStateInner::Issued { - collect_notifier, - barrier_inflight_latency: timer, - .. - } => { - timer.observe_duration(); - collect_notifier - } - _ => unreachable!(), - }; + must_match!(prev_state, ManagedBarrierStateInner::Issued { + barrier_inflight_latency: timer, + .. + } => { + timer.observe_duration(); + }); let create_mview_progress = self .create_mview_progress @@ -161,46 +232,42 @@ impl ManagedBarrierState { } } - if let Some(notifier) = collect_notifier { - // Notify about barrier finishing. - let result = CollectResult { - create_mview_progress, - kind, - }; - - if notifier.send(Ok(result)).is_err() { - warn!( - "failed to notify barrier collection with epoch {}", - prev_epoch - ) - } - } + self.await_epoch_completed_futures.push_back( + sync_epoch(&self.state_store, &self.streaming_metrics, prev_epoch, kind).map( + move |result| { + ( + prev_epoch, + result.map(move |sync_result| BarrierCompleteResult { + sync_result, + create_mview_progress, + }), + ) + }, + ), + ); } } - /// Returns an iterator on the notifiers of epochs that is awaiting on `actor_id`. + /// Returns an iterator on epochs that is awaiting on `actor_id`. /// This is used on notifying actor failure. On actor failure, the - /// barrier manager can call this method to iterate on notifiers of epochs that + /// barrier manager can call this method to iterate on epochs that /// waits on the failed actor and then notify failure on the result /// sender of the epoch. - pub(crate) fn notifiers_await_on_actor( - &mut self, + pub(crate) fn epochs_await_on_actor( + &self, actor_id: ActorId, - ) -> impl Iterator>)> + '_ { + ) -> impl Iterator + '_ { self.epoch_barrier_state_map - .iter_mut() + .iter() .filter_map(move |(prev_epoch, barrier_state)| { #[allow(clippy::single_match)] - match &mut barrier_state.inner { + match barrier_state.inner { ManagedBarrierStateInner::Issued { ref remaining_actors, - ref mut collect_notifier, .. } => { if remaining_actors.contains(&actor_id) { - collect_notifier - .take() - .map(|notifier| (*prev_epoch, notifier)) + Some(*prev_epoch) } else { None } @@ -247,7 +314,13 @@ impl ManagedBarrierState { actor_id, barrier.epoch.curr ); assert_eq!(curr_epoch, barrier.epoch.curr); - self.may_notify(barrier.epoch.prev); + self.may_have_collected_all(barrier.epoch.prev); + } + Some(BarrierState { inner, .. }) => { + panic!( + "cannot collect new actor barrier {:?} at current state: {:?}", + barrier.epoch, inner + ) } None => { self.epoch_barrier_state_map.insert( @@ -270,7 +343,6 @@ impl ManagedBarrierState { &mut self, barrier: &Barrier, actor_ids_to_collect: HashSet, - collect_notifier: oneshot::Sender>, ) { let timer = self .streaming_metrics @@ -294,22 +366,17 @@ impl ManagedBarrierState { ManagedBarrierStateInner::Issued { remaining_actors, barrier_inflight_latency: timer, - collect_notifier: Some(collect_notifier), } } - Some(&mut BarrierState { - inner: ManagedBarrierStateInner::Issued { .. }, - .. - }) => { + Some(BarrierState { ref inner, .. }) => { panic!( - "barrier epochs{:?} state has already been `Issued`", - barrier.epoch + "barrier epochs{:?} state has already been `Issued`. Current state: {:?}", + barrier.epoch, inner ); } None => ManagedBarrierStateInner::Issued { remaining_actors: actor_ids_to_collect, barrier_inflight_latency: timer, - collect_notifier: Some(collect_notifier), }, }; self.epoch_barrier_state_map.insert( @@ -320,7 +387,66 @@ impl ManagedBarrierState { kind: barrier.kind, }, ); - self.may_notify(barrier.epoch.prev); + self.may_have_collected_all(barrier.epoch.prev); + } + + /// Return a future that yields the next completed epoch. The future is cancellation safe. + pub(crate) fn next_completed_epoch(&mut self) -> impl Future + '_ { + pending_on_none(self.await_epoch_completed_futures.next()).map(|(prev_epoch, result)| { + let state = self + .epoch_barrier_state_map + .get_mut(&prev_epoch) + .expect("should exist"); + // sanity check on barrier state + assert_matches!(&state.inner, ManagedBarrierStateInner::AllCollected); + state.inner = ManagedBarrierStateInner::Completed(result); + prev_epoch + }) + } + + /// Pop the completion result of an completed epoch. + /// Return: + /// - `Err(_)` `prev_epoch` is not an epoch to be collected. + /// - `Ok(None)` when `prev_epoch` exists but has not completed. + /// - `Ok(Some(_))` when `prev_epoch` has completed but not been reclaimed yet. + /// The `BarrierCompleteResult` will be popped out. + pub(crate) fn pop_completed_epoch( + &mut self, + prev_epoch: u64, + ) -> StreamResult>> { + let state = self + .epoch_barrier_state_map + .get(&prev_epoch) + .ok_or_else(|| { + // It's still possible that `collect_complete_receiver` does not contain the target epoch + // when receiving collect_barrier request. Because `collect_complete_receiver` could + // be cleared when CN is under recovering. We should return error rather than panic. + anyhow!( + "barrier collect complete receiver for prev epoch {} not exists", + prev_epoch + ) + })?; + match &state.inner { + ManagedBarrierStateInner::Completed(_) => { + match self + .epoch_barrier_state_map + .remove(&prev_epoch) + .expect("should exists") + .inner + { + ManagedBarrierStateInner::Completed(result) => Ok(Some(result)), + _ => unreachable!(), + } + } + _ => Ok(None), + } + } + + #[cfg(test)] + async fn pop_next_completed_epoch(&mut self) -> u64 { + let epoch = self.next_completed_epoch().await; + let _ = self.pop_completed_epoch(epoch).unwrap().unwrap(); + epoch } } @@ -329,7 +455,6 @@ mod tests { use std::collections::HashSet; use risingwave_common::util::epoch::test_epoch; - use tokio::sync::oneshot; use crate::executor::Barrier; use crate::task::barrier_manager::managed_state::ManagedBarrierState; @@ -340,17 +465,18 @@ mod tests { let barrier1 = Barrier::new_test_barrier(test_epoch(1)); let barrier2 = Barrier::new_test_barrier(test_epoch(2)); let barrier3 = Barrier::new_test_barrier(test_epoch(3)); - let (tx1, _rx1) = oneshot::channel(); - let (tx2, _rx2) = oneshot::channel(); - let (tx3, _rx3) = oneshot::channel(); let actor_ids_to_collect1 = HashSet::from([1, 2]); let actor_ids_to_collect2 = HashSet::from([1, 2]); let actor_ids_to_collect3 = HashSet::from([1, 2, 3]); - managed_barrier_state.transform_to_issued(&barrier1, actor_ids_to_collect1, tx1); - managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2, tx2); - managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3, tx3); + managed_barrier_state.transform_to_issued(&barrier1, actor_ids_to_collect1); + managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2); + managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3); managed_barrier_state.collect(1, &barrier1); managed_barrier_state.collect(2, &barrier1); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(0) + ); assert_eq!( managed_barrier_state .epoch_barrier_state_map @@ -362,6 +488,10 @@ mod tests { managed_barrier_state.collect(1, &barrier2); managed_barrier_state.collect(1, &barrier3); managed_barrier_state.collect(2, &barrier2); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(1) + ); assert_eq!( managed_barrier_state .epoch_barrier_state_map @@ -372,6 +502,10 @@ mod tests { ); managed_barrier_state.collect(2, &barrier3); managed_barrier_state.collect(3, &barrier3); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(2) + ); assert!(managed_barrier_state.epoch_barrier_state_map.is_empty()); } @@ -381,15 +515,12 @@ mod tests { let barrier1 = Barrier::new_test_barrier(test_epoch(1)); let barrier2 = Barrier::new_test_barrier(test_epoch(2)); let barrier3 = Barrier::new_test_barrier(test_epoch(3)); - let (tx1, _rx1) = oneshot::channel(); - let (tx2, _rx2) = oneshot::channel(); - let (tx3, _rx3) = oneshot::channel(); let actor_ids_to_collect1 = HashSet::from([1, 2, 3, 4]); let actor_ids_to_collect2 = HashSet::from([1, 2, 3]); let actor_ids_to_collect3 = HashSet::from([1, 2]); - managed_barrier_state.transform_to_issued(&barrier1, actor_ids_to_collect1, tx1); - managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2, tx2); - managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3, tx3); + managed_barrier_state.transform_to_issued(&barrier1, actor_ids_to_collect1); + managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2); + managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3); managed_barrier_state.collect(1, &barrier1); managed_barrier_state.collect(1, &barrier2); @@ -416,6 +547,18 @@ mod tests { &0 ); managed_barrier_state.collect(4, &barrier1); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(0) + ); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(1) + ); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(2) + ); assert!(managed_barrier_state.epoch_barrier_state_map.is_empty()); } @@ -425,9 +568,6 @@ mod tests { let barrier1 = Barrier::new_test_barrier(test_epoch(1)); let barrier2 = Barrier::new_test_barrier(test_epoch(2)); let barrier3 = Barrier::new_test_barrier(test_epoch(3)); - let (tx1, _rx1) = oneshot::channel(); - let (tx2, _rx2) = oneshot::channel(); - let (tx3, _rx3) = oneshot::channel(); let actor_ids_to_collect1 = HashSet::from([1, 2]); let actor_ids_to_collect2 = HashSet::from([1, 2, 3]); let actor_ids_to_collect3 = HashSet::from([1, 2, 3]); @@ -462,7 +602,11 @@ mod tests { managed_barrier_state.collect(2, &barrier1); managed_barrier_state.collect(2, &barrier2); managed_barrier_state.collect(2, &barrier3); - managed_barrier_state.transform_to_issued(&barrier1, actor_ids_to_collect1, tx1); + managed_barrier_state.transform_to_issued(&barrier1, actor_ids_to_collect1); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(0) + ); assert_eq!( managed_barrier_state .epoch_barrier_state_map @@ -471,8 +615,12 @@ mod tests { .0, { &test_epoch(1) } ); - managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2, tx2); + managed_barrier_state.transform_to_issued(&barrier2, actor_ids_to_collect2); managed_barrier_state.collect(3, &barrier2); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(1) + ); assert_eq!( managed_barrier_state .epoch_barrier_state_map @@ -490,7 +638,11 @@ mod tests { .0, { &test_epoch(2) } ); - managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3, tx3); + managed_barrier_state.transform_to_issued(&barrier3, actor_ids_to_collect3); + assert_eq!( + managed_barrier_state.pop_next_completed_epoch().await, + test_epoch(2) + ); assert!(managed_barrier_state.epoch_barrier_state_map.is_empty()); } } diff --git a/src/stream/src/task/barrier_manager/progress.rs b/src/stream/src/task/barrier_manager/progress.rs index 2b83050a57d9a..476534967072b 100644 --- a/src/stream/src/task/barrier_manager/progress.rs +++ b/src/stream/src/task/barrier_manager/progress.rs @@ -15,13 +15,13 @@ use super::LocalBarrierManager; use crate::task::barrier_manager::LocalBarrierEvent::ReportCreateProgress; use crate::task::barrier_manager::LocalBarrierWorker; -use crate::task::{ActorId, SharedContext}; +use crate::task::ActorId; type ConsumedEpoch = u64; type ConsumedRows = u64; #[derive(Debug, Clone, Copy)] -pub(super) enum BackfillState { +pub(crate) enum BackfillState { ConsumingUpstream(ConsumedEpoch, ConsumedRows), Done(ConsumedRows), } @@ -161,7 +161,7 @@ impl CreateMviewProgress { } } -impl SharedContext { +impl LocalBarrierManager { /// Create a struct for reporting the progress of creating mview. The backfill executors should /// report the progress of barrier rearranging continuously using this. The updated progress /// will be collected by the local barrier manager and reported to the meta service in this @@ -174,6 +174,6 @@ impl SharedContext { backfill_actor_id: ActorId, ) -> CreateMviewProgress { trace!("register create mview progress: {}", backfill_actor_id); - CreateMviewProgress::new(self.barrier_manager.clone(), backfill_actor_id) + CreateMviewProgress::new(self.clone(), backfill_actor_id) } } diff --git a/src/stream/src/task/barrier_manager/tests.rs b/src/stream/src/task/barrier_manager/tests.rs index db65b48bb97ca..ae248facc0e5f 100644 --- a/src/stream/src/task/barrier_manager/tests.rs +++ b/src/stream/src/task/barrier_manager/tests.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::future::{poll_fn, Future}; use std::iter::once; +use std::pin::pin; +use std::task::Poll; use itertools::Itertools; use risingwave_common::util::epoch::test_epoch; @@ -22,7 +25,7 @@ use super::*; #[tokio::test] async fn test_managed_barrier_collection() -> StreamResult<()> { - let manager = LocalBarrierManager::for_test(); + let (actor_op_tx, manager) = LocalBarrierManager::spawn_for_test().await; let register_sender = |actor_id: u32| { let (barrier_tx, barrier_rx) = unbounded_channel(); @@ -44,11 +47,10 @@ async fn test_managed_barrier_collection() -> StreamResult<()> { let barrier = Barrier::new_test_barrier(curr_epoch); let epoch = barrier.epoch.prev; - manager + actor_op_tx .send_barrier(barrier.clone(), actor_ids.clone(), actor_ids) .await .unwrap(); - let mut complete_receiver = manager.remove_collect_rx(barrier.epoch.prev).await?; // Collect barriers from actors let collected_barriers = rxs .iter_mut() @@ -59,16 +61,14 @@ async fn test_managed_barrier_collection() -> StreamResult<()> { }) .collect_vec(); + let mut await_epoch_future = pin!(actor_op_tx.await_epoch_completed(epoch)); + // Report to local barrier manager for (i, (actor_id, barrier)) in collected_barriers.into_iter().enumerate() { manager.collect(actor_id, &barrier); manager.flush_all_events().await; - let notified = complete_receiver - .complete_receiver - .as_mut() - .unwrap() - .try_recv() - .is_ok(); + let notified = + poll_fn(|cx| Poll::Ready(await_epoch_future.as_mut().poll(cx).is_ready())).await; assert_eq!(notified, i == count - 1); } @@ -77,7 +77,7 @@ async fn test_managed_barrier_collection() -> StreamResult<()> { #[tokio::test] async fn test_managed_barrier_collection_before_send_request() -> StreamResult<()> { - let manager = LocalBarrierManager::for_test(); + let (actor_op_tx, manager) = LocalBarrierManager::spawn_for_test().await; let register_sender = |actor_id: u32| { let (barrier_tx, barrier_rx) = unbounded_channel(); @@ -110,11 +110,10 @@ async fn test_managed_barrier_collection_before_send_request() -> StreamResult<( manager.collect(extra_actor_id, &barrier); // Send the barrier to all actors - manager + actor_op_tx .send_barrier(barrier.clone(), actor_ids_to_send, actor_ids_to_collect) .await .unwrap(); - let mut complete_receiver = manager.remove_collect_rx(barrier.epoch.prev).await?; // Collect barriers from actors let collected_barriers = rxs @@ -126,16 +125,14 @@ async fn test_managed_barrier_collection_before_send_request() -> StreamResult<( }) .collect_vec(); + let mut await_epoch_future = pin!(actor_op_tx.await_epoch_completed(epoch)); + // Report to local barrier manager for (i, (actor_id, barrier)) in collected_barriers.into_iter().enumerate() { manager.collect(actor_id, &barrier); manager.flush_all_events().await; - let notified = complete_receiver - .complete_receiver - .as_mut() - .unwrap() - .try_recv() - .is_ok(); + let notified = + poll_fn(|cx| Poll::Ready(await_epoch_future.as_mut().poll(cx).is_ready())).await; assert_eq!(notified, i == count - 1); } diff --git a/src/stream/src/task/env.rs b/src/stream/src/task/env.rs index 0083e8e02ff28..0c5a299e8487d 100644 --- a/src/stream/src/task/env.rs +++ b/src/stream/src/task/env.rs @@ -20,8 +20,8 @@ use risingwave_common::system_param::local_manager::LocalSystemParamsManagerRef; use risingwave_common::util::addr::HostAddr; use risingwave_connector::source::monitor::SourceMetrics; use risingwave_connector::ConnectorParams; +use risingwave_dml::dml_manager::DmlManagerRef; use risingwave_rpc_client::MetaClient; -use risingwave_source::dml_manager::DmlManagerRef; use risingwave_storage::StateStoreImpl; pub(crate) type WorkerNodeId = u32; @@ -92,8 +92,8 @@ impl StreamEnvironment { #[cfg(test)] pub fn for_test() -> Self { use risingwave_common::system_param::local_manager::LocalSystemParamsManager; + use risingwave_dml::dml_manager::DmlManager; use risingwave_pb::connector_service::SinkPayloadFormat; - use risingwave_source::dml_manager::DmlManager; use risingwave_storage::monitor::MonitoredStorageMetrics; StreamEnvironment { server_addr: "127.0.0.1:5688".parse().unwrap(), diff --git a/src/stream/src/task/mod.rs b/src/stream/src/task/mod.rs index b1f1da526bcc5..7a6fd40f9231a 100644 --- a/src/stream/src/task/mod.rs +++ b/src/stream/src/task/mod.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::collections::HashMap; -use std::sync::Arc; use anyhow::anyhow; use parking_lot::{MappedMutexGuard, Mutex, MutexGuard, RwLock}; @@ -31,11 +30,8 @@ mod stream_manager; pub use barrier_manager::*; pub use env::*; -use risingwave_storage::StateStoreImpl; pub use stream_manager::*; -use crate::executor::monitor::StreamingMetrics; - pub type ConsumableChannelPair = (Option, Option); pub type ActorId = u32; pub type FragmentId = u32; @@ -44,6 +40,11 @@ pub type UpDownActorIds = (ActorId, ActorId); pub type UpDownFragmentIds = (FragmentId, FragmentId); /// Stores the information which may be modified from the data plane. +/// +/// The data structure is created in `LocalBarrierWorker` and is shared by actors created +/// between two recoveries. In every recovery, the `LocalBarrierWorker` will create a new instance of +/// `SharedContext`, and the original one becomes stale. The new one is shared by actors created after +/// recovery. pub struct SharedContext { /// Stores the senders and receivers for later `Processor`'s usage. /// @@ -79,9 +80,9 @@ pub struct SharedContext { // disconnected. pub(crate) compute_client_pool: ComputeClientPool, - pub(crate) barrier_manager: LocalBarrierManager, - pub(crate) config: StreamingConfig, + + pub(super) local_barrier_manager: LocalBarrierManager, } impl std::fmt::Debug for SharedContext { @@ -95,17 +96,16 @@ impl std::fmt::Debug for SharedContext { impl SharedContext { pub fn new( addr: HostAddr, - state_store: StateStoreImpl, config: &StreamingConfig, - streaming_metrics: Arc, + local_barrier_manager: LocalBarrierManager, ) -> Self { Self { channel_map: Default::default(), actor_infos: Default::default(), addr, compute_client_pool: ComputeClientPool::default(), - barrier_manager: LocalBarrierManager::new(state_store, streaming_metrics), config: config.clone(), + local_barrier_manager, } } @@ -118,7 +118,6 @@ impl SharedContext { actor_infos: Default::default(), addr: LOCAL_TEST_ADDR.clone(), compute_client_pool: ComputeClientPool::default(), - barrier_manager: LocalBarrierManager::for_test(), config: StreamingConfig { developer: StreamingDeveloperConfig { exchange_initial_permits: permit::for_test::INITIAL_PERMITS, @@ -128,13 +127,10 @@ impl SharedContext { }, ..Default::default() }, + local_barrier_manager: LocalBarrierManager::for_test(), } } - pub fn barrier_manager(&self) -> &LocalBarrierManager { - &self.barrier_manager - } - /// Get the channel pair for the given actor ids. If the channel pair does not exist, create one /// with the configured permits. fn get_or_insert_channels( @@ -160,26 +156,13 @@ impl SharedContext { .ok_or_else(|| anyhow!("sender for {ids:?} has already been taken").into()) } - pub fn take_receiver(&self, ids: &UpDownActorIds) -> StreamResult { - self.get_or_insert_channels(*ids) + pub fn take_receiver(&self, ids: UpDownActorIds) -> StreamResult { + self.get_or_insert_channels(ids) .1 .take() .ok_or_else(|| anyhow!("receiver for {ids:?} has already been taken").into()) } - pub fn retain_channel(&self, mut f: F) - where - F: FnMut(&(u32, u32)) -> bool, - { - self.channel_map - .lock() - .retain(|up_down_ids, _| f(up_down_ids)); - } - - pub fn clear_channels(&self) { - self.channel_map.lock().clear() - } - pub fn get_actor_info(&self, actor_id: &ActorId) -> StreamResult { self.actor_infos .read() @@ -187,6 +170,20 @@ impl SharedContext { .cloned() .ok_or_else(|| anyhow!("actor {} not found in info table", actor_id).into()) } + + pub fn config(&self) -> &StreamingConfig { + &self.config + } + + pub fn drop_actors(&self, actors: &[ActorId]) { + self.channel_map + .lock() + .retain(|(up_id, _), _| !actors.contains(up_id)); + let mut actor_infos = self.actor_infos.write(); + for actor_id in actors { + actor_infos.remove(actor_id); + } + } } /// Generate a globally unique executor id. diff --git a/src/stream/src/task/stream_manager.rs b/src/stream/src/task/stream_manager.rs index db60f43ea95dd..ec634f7068128 100644 --- a/src/stream/src/task/stream_manager.rs +++ b/src/stream/src/task/stream_manager.rs @@ -13,96 +13,66 @@ // limitations under the License. use core::time::Duration; -use std::collections::{HashMap, HashSet}; +use std::collections::HashMap; use std::fmt::Debug; use std::io::Write; +use std::mem::take; use std::sync::atomic::AtomicU64; use std::sync::Arc; -use anyhow::{anyhow, Context}; +use anyhow::anyhow; use async_recursion::async_recursion; use futures::FutureExt; -use hytra::TrAdder; use itertools::Itertools; +use parking_lot::Mutex; use risingwave_common::bail; use risingwave_common::buffer::Bitmap; use risingwave_common::catalog::{Field, Schema}; -use risingwave_common::config::{MetricLevel, StreamingConfig}; -use risingwave_common::util::addr::HostAddr; -use risingwave_common::util::runtime::BackgroundShutdownRuntime; +use risingwave_common::config::MetricLevel; use risingwave_pb::common::ActorInfo; use risingwave_pb::stream_plan; -use risingwave_pb::stream_plan::barrier::BarrierKind; use risingwave_pb::stream_plan::stream_node::NodeBody; -use risingwave_pb::stream_plan::StreamNode; +use risingwave_pb::stream_plan::{StreamActor, StreamNode}; use risingwave_storage::monitor::HummockTraceFutureExt; -use risingwave_storage::store::SyncResult; -use risingwave_storage::{dispatch_state_store, StateStore, StateStoreImpl}; +use risingwave_storage::{dispatch_state_store, StateStore}; +use rw_futures_util::AttachedFuture; use thiserror_ext::AsReport; -use tokio::sync::Mutex; +use tokio::sync::mpsc::unbounded_channel; +use tokio::sync::oneshot; use tokio::task::JoinHandle; -use super::{unique_executor_id, unique_operator_id, CollectResult}; +use super::{unique_executor_id, unique_operator_id, BarrierCompleteResult}; use crate::error::StreamResult; use crate::executor::exchange::permit::Receiver; use crate::executor::monitor::StreamingMetrics; use crate::executor::subtask::SubtaskHandle; -use crate::executor::*; +use crate::executor::{ + Actor, ActorContext, ActorContextRef, Barrier, DispatchExecutor, DispatcherImpl, Executor, + ExecutorInfo, WrapperExecutor, +}; use crate::from_proto::create_executor; -use crate::task::{ActorId, FragmentId, SharedContext, StreamEnvironment, UpDownActorIds}; +use crate::task::barrier_manager::{EventSender, LocalActorOperation, LocalBarrierWorker}; +use crate::task::{ + ActorId, FragmentId, LocalBarrierManager, SharedContext, StreamActorManager, + StreamActorManagerState, StreamEnvironment, UpDownActorIds, +}; #[cfg(test)] -pub static LOCAL_TEST_ADDR: std::sync::LazyLock = +pub static LOCAL_TEST_ADDR: std::sync::LazyLock = std::sync::LazyLock::new(|| "127.0.0.1:2333".parse().unwrap()); pub type ActorHandle = JoinHandle<()>; pub type AtomicU64Ref = Arc; -pub struct LocalStreamManagerCore { - /// Runtime for the streaming actors. - runtime: BackgroundShutdownRuntime, - - /// Each processor runs in a future. Upon receiving a `Terminate` message, they will exit. - /// `handles` store join handles of these futures, and therefore we could wait their - /// termination. - handles: HashMap, - - pub(crate) context: Arc, - - /// Stores all actor information, taken after actor built. - actors: HashMap, - - /// Stores all actor tokio runtime monitoring tasks. - actor_monitor_tasks: HashMap, - - /// The state store implement - state_store: StateStoreImpl, - - /// Metrics of the stream manager - pub(crate) streaming_metrics: Arc, - - /// Config of streaming engine - pub(crate) config: StreamingConfig, - - /// Manages the await-trees of all actors. - await_tree_reg: Option>, - - /// Watermark epoch number. - watermark_epoch: AtomicU64Ref, - - total_mem_val: Arc>, -} - /// `LocalStreamManager` manages all stream executors in this project. +#[derive(Clone)] pub struct LocalStreamManager { - core: Mutex, + await_tree_reg: Option>>>, - // Maintain a copy of the core to reduce async locks - state_store: StateStoreImpl, - context: Arc, + pub env: StreamEnvironment, - total_mem_val: Arc>, + actor_op_tx: EventSender, } /// Report expression evaluation errors to the actor context. @@ -137,7 +107,7 @@ pub struct ExecutorParams { pub op_info: String, /// The input executor. - pub input: Vec, + pub input: Vec, /// FragmentId of the actor pub fragment_id: FragmentId, @@ -153,6 +123,13 @@ pub struct ExecutorParams { /// Used for reporting expression evaluation errors. pub eval_error_report: ActorEvalErrorReport, + + /// `watermark_epoch` field in `MemoryManager` + pub watermark_epoch: AtomicU64Ref, + + pub shared_context: Arc, + + pub local_barrier_manager: LocalBarrierManager, } impl Debug for ExecutorParams { @@ -169,34 +146,29 @@ impl Debug for ExecutorParams { } impl LocalStreamManager { - fn with_core(core: LocalStreamManagerCore) -> Self { - Self { - state_store: core.state_store.clone(), - context: core.context.clone(), - total_mem_val: core.total_mem_val.clone(), - core: Mutex::new(core), - } - } - pub fn new( - addr: HostAddr, - state_store: StateStoreImpl, + env: StreamEnvironment, streaming_metrics: Arc, - config: StreamingConfig, await_tree_config: Option, + watermark_epoch: AtomicU64Ref, ) -> Self { - Self::with_core(LocalStreamManagerCore::new( - addr, - state_store, - streaming_metrics, - config, - await_tree_config, - )) - } + let await_tree_reg = + await_tree_config.map(|config| Arc::new(Mutex::new(await_tree::Registry::new(config)))); + + let (actor_op_tx, actor_op_rx) = unbounded_channel(); - #[cfg(test)] - pub fn for_test() -> Self { - Self::with_core(LocalStreamManagerCore::for_test()) + let _join_handle = LocalBarrierWorker::spawn( + env.clone(), + streaming_metrics, + await_tree_reg.clone(), + watermark_epoch, + actor_op_rx, + ); + Self { + await_tree_reg, + env, + actor_op_tx: EventSender(actor_op_tx), + } } /// Print the traces of all actors periodically, used for debugging only. @@ -204,13 +176,13 @@ impl LocalStreamManager { tokio::spawn(async move { loop { tokio::time::sleep(std::time::Duration::from_millis(5000)).await; - let mut core = self.core.lock().await; let mut o = std::io::stdout().lock(); - for (k, trace) in core + for (k, trace) in self .await_tree_reg - .as_mut() + .as_ref() .expect("async stack trace not enabled") + .lock() .iter() { writeln!(o, ">> Actor {}\n\n{}", k, trace).ok(); @@ -220,19 +192,13 @@ impl LocalStreamManager { } /// Get await-tree contexts for all actors. - pub async fn get_actor_traces(&self) -> HashMap { - let core = self.core.lock().await; - match &core.await_tree_reg { - Some(mgr) => mgr.iter().map(|(k, v)| (*k, v)).collect(), + pub fn get_actor_traces(&self) -> HashMap { + match &self.await_tree_reg.as_ref() { + Some(mgr) => mgr.lock().iter().map(|(k, v)| (*k, v)).collect(), None => Default::default(), } } - /// Get all existing actor ids. - pub async fn all_actor_ids(&self) -> HashSet { - self.core.lock().await.handles.keys().cloned().collect() - } - /// Broadcast a barrier to all senders. Save a receiver in barrier manager pub async fn send_barrier( &self, @@ -240,213 +206,173 @@ impl LocalStreamManager { actor_ids_to_send: impl IntoIterator, actor_ids_to_collect: impl IntoIterator, ) -> StreamResult<()> { - if barrier.kind == BarrierKind::Initial { - let core = self.core.lock().await; - core.get_watermark_epoch() - .store(barrier.epoch.curr, std::sync::atomic::Ordering::SeqCst); - } - let barrier_manager = self.context.barrier_manager(); - barrier_manager + self.actor_op_tx .send_barrier(barrier, actor_ids_to_send, actor_ids_to_collect) - .await?; - Ok(()) - } - - /// Reset the state of the barrier manager. - pub fn reset_barrier_manager(&self) { - self.context.barrier_manager().reset(); + .await } /// Use `epoch` to find collect rx. And wait for all actor to be collected before /// returning. - pub async fn collect_barrier(&self, epoch: u64) -> StreamResult { - let complete_receiver = { - let barrier_manager = self.context.barrier_manager(); - barrier_manager.remove_collect_rx(epoch).await? - }; - // Wait for all actors finishing this barrier. - let result = complete_receiver - .complete_receiver - .expect("no rx for local mode") + pub async fn collect_barrier(&self, prev_epoch: u64) -> StreamResult { + self.actor_op_tx.await_epoch_completed(prev_epoch).await + } + + /// Drop the resources of the given actors. + pub async fn drop_actors(&self, actors: Vec) -> StreamResult<()> { + self.actor_op_tx + .send_and_await(|result_sender| LocalActorOperation::DropActors { + actors, + result_sender, + }) .await - .context("failed to collect barrier")??; - Ok(result) } - pub async fn sync_epoch(&self, epoch: u64) -> StreamResult { - let timer = self - .core - .lock() + /// Force stop all actors on this worker, and then drop their resources. + pub async fn reset(&self, prev_epoch: u64) { + self.actor_op_tx + .send_and_await(|result_sender| LocalActorOperation::Reset { + result_sender, + prev_epoch, + }) .await - .streaming_metrics - .barrier_sync_latency - .start_timer(); - let res = dispatch_state_store!(self.state_store.clone(), store, { - store.sync(epoch).await.map_err(|e| { - tracing::error!( - epoch, - error = %e.as_report(), - "Failed to sync state store", - ); - e.into() + .expect("should receive reset") + } + + pub async fn update_actors(&self, actors: Vec) -> StreamResult<()> { + self.actor_op_tx + .send_and_await(|result_sender| LocalActorOperation::UpdateActors { + actors, + result_sender, }) - }); - timer.observe_duration(); - res + .await? } - pub async fn clear_storage_buffer(&self) { - dispatch_state_store!(self.state_store.clone(), store, { - store.clear_shared_buffer().await.unwrap(); - }); + pub async fn build_actors(&self, actors: Vec) -> StreamResult<()> { + self.actor_op_tx + .send_and_await(|result_sender| LocalActorOperation::BuildActors { + actors, + result_sender, + }) + .await? + } + + pub async fn update_actor_info(&self, new_actor_infos: Vec) -> StreamResult<()> { + self.actor_op_tx + .send_and_await(|result_sender| LocalActorOperation::UpdateActorInfo { + new_actor_infos, + result_sender, + }) + .await? } + pub async fn take_receiver(&self, ids: UpDownActorIds) -> StreamResult { + self.actor_op_tx + .send_and_await(|result_sender| LocalActorOperation::TakeReceiver { + ids, + result_sender, + }) + .await? + } +} + +impl LocalBarrierWorker { /// Drop the resources of the given actors. - pub async fn drop_actors(&self, actors: &[ActorId]) -> StreamResult<()> { - let mut core = self.core.lock().await; + pub(super) fn drop_actors(&mut self, actors: &[ActorId]) { + self.current_shared_context.drop_actors(actors); for &id in actors { - core.drop_actor(id); + self.actor_manager_state.drop_actor(id); } tracing::debug!(actors = ?actors, "drop actors"); - Ok(()) } /// Force stop all actors on this worker, and then drop their resources. - pub async fn stop_all_actors(&self) -> StreamResult<()> { - self.core.lock().await.stop_all_actors().await; - self.reset_barrier_manager(); - // Clear shared buffer in storage to release memory - self.clear_storage_buffer().await; - - Ok(()) - } - - pub async fn take_receiver(&self, ids: UpDownActorIds) -> StreamResult { - let core = self.core.lock().await; - core.context.take_receiver(&ids) - } - - pub async fn update_actors(&self, actors: &[stream_plan::StreamActor]) -> StreamResult<()> { - let mut core = self.core.lock().await; - core.update_actors(actors) + pub(super) async fn reset(&mut self, prev_epoch: u64) { + let actor_handles = self.actor_manager_state.drain_actor_handles(); + for (actor_id, handle) in &actor_handles { + tracing::debug!("force stopping actor {}", actor_id); + handle.abort(); + } + for (actor_id, handle) in actor_handles { + tracing::debug!("join actor {}", actor_id); + let result = handle.await; + assert!(result.is_ok() || result.unwrap_err().is_cancelled()); + } + // Clear the join handle of creating actors + for handle in take(&mut self.actor_manager_state.creating_actors) + .into_iter() + .map(|attached_future| attached_future.into_inner().0) + { + handle.abort(); + let result = handle.await; + assert!(result.is_ok() || result.err().unwrap().is_cancelled()); + } + self.actor_manager_state.clear_state(); + if let Some(m) = self.actor_manager.await_tree_reg.as_ref() { + m.lock().clear(); + } + dispatch_state_store!(&self.actor_manager.env.state_store(), store, { + store.clear_shared_buffer(prev_epoch).await; + }); + self.reset_state(); + self.actor_manager.env.dml_manager_ref().clear(); } - /// This function could only be called once during the lifecycle of `LocalStreamManager` for - /// now. - pub async fn update_actor_info(&self, actor_infos: &[ActorInfo]) -> StreamResult<()> { - let mut core = self.core.lock().await; - core.update_actor_info(actor_infos) + pub(super) fn update_actors( + &mut self, + actors: Vec, + ) -> StreamResult<()> { + self.actor_manager_state.update_actors(actors) } /// This function could only be called once during the lifecycle of `LocalStreamManager` for /// now. - pub async fn build_actors( - &self, + pub(super) fn start_create_actors( + &mut self, actors: &[ActorId], - env: StreamEnvironment, - ) -> StreamResult<()> { - let mut core = self.core.lock().await; - core.build_actors(actors, env).await - } - - pub async fn config(&self) -> StreamingConfig { - let core = self.core.lock().await; - core.config.clone() - } - - /// After memory manager is created, it will store the watermark epoch in stream manager, so - /// stream executor can get it to build managed cache. - pub async fn set_watermark_epoch(&self, watermark_epoch: AtomicU64Ref) { - let mut guard = self.core.lock().await; - guard.watermark_epoch = watermark_epoch; - } - - pub fn total_mem_usage(&self) -> usize { - self.total_mem_val.get() as usize - } -} - -impl LocalStreamManagerCore { - fn new( - addr: HostAddr, - state_store: StateStoreImpl, - streaming_metrics: Arc, - config: StreamingConfig, - await_tree_config: Option, - ) -> Self { - let context = SharedContext::new( - addr, - state_store.clone(), - &config, - streaming_metrics.clone(), - ); - Self::new_inner( - state_store, - context, - streaming_metrics, - config, - await_tree_config, - ) - } - - fn new_inner( - state_store: StateStoreImpl, - context: SharedContext, - streaming_metrics: Arc, - config: StreamingConfig, - await_tree_config: Option, - ) -> Self { - let runtime = { - let mut builder = tokio::runtime::Builder::new_multi_thread(); - if let Some(worker_threads_num) = config.actor_runtime_worker_threads_num { - builder.worker_threads(worker_threads_num); + result_sender: oneshot::Sender>, + ) { + let actors = { + let actor_result = actors + .iter() + .map(|actor_id| { + self.actor_manager_state + .actors + .remove(actor_id) + .ok_or_else(|| anyhow!("No such actor with actor id:{}", actor_id)) + }) + .try_collect(); + match actor_result { + Ok(actors) => actors, + Err(e) => { + let _ = result_sender.send(Err(e.into())); + return; + } } - builder - .thread_name("rw-streaming") - .enable_all() - .build() - .unwrap() }; - - Self { - runtime: runtime.into(), - handles: HashMap::new(), - context: Arc::new(context), - actors: HashMap::new(), - actor_monitor_tasks: HashMap::new(), - state_store, - streaming_metrics, - config, - await_tree_reg: await_tree_config.map(await_tree::Registry::new), - watermark_epoch: Arc::new(AtomicU64::new(0)), - total_mem_val: Arc::new(TrAdder::new()), - } - } - - #[cfg(test)] - fn for_test() -> Self { - use risingwave_storage::monitor::MonitoredStorageMetrics; - let streaming_metrics = Arc::new(StreamingMetrics::unused()); - Self::new_inner( - StateStoreImpl::shared_in_memory_store(Arc::new(MonitoredStorageMetrics::unused())), - SharedContext::for_test(), - streaming_metrics, - StreamingConfig::default(), - None, - ) + let actor_manager = self.actor_manager.clone(); + let join_handle = self + .actor_manager + .runtime + .spawn(actor_manager.create_actors(actors, self.current_shared_context.clone())); + self.actor_manager_state + .creating_actors + .push(AttachedFuture::new(join_handle, result_sender)); } +} +impl StreamActorManager { /// Create dispatchers with downstream information registered before fn create_dispatcher( - &mut self, - input: BoxedExecutor, + &self, + input: Executor, dispatchers: &[stream_plan::Dispatcher], actor_id: ActorId, fragment_id: FragmentId, + shared_context: &Arc, ) -> StreamResult { let dispatcher_impls = dispatchers .iter() - .map(|dispatcher| DispatcherImpl::new(&self.context, actor_id, dispatcher)) + .map(|dispatcher| DispatcherImpl::new(shared_context, actor_id, dispatcher)) .try_collect()?; Ok(DispatchExecutor::new( @@ -454,7 +380,7 @@ impl LocalStreamManagerCore { dispatcher_impls, actor_id, fragment_id, - self.context.clone(), + shared_context.clone(), self.streaming_metrics.clone(), )) } @@ -463,7 +389,7 @@ impl LocalStreamManagerCore { #[allow(clippy::too_many_arguments)] #[async_recursion] async fn create_nodes_inner( - &mut self, + &self, fragment_id: FragmentId, node: &stream_plan::StreamNode, env: StreamEnvironment, @@ -472,7 +398,8 @@ impl LocalStreamManagerCore { vnode_bitmap: Option, has_stateful: bool, subtasks: &mut Vec, - ) -> StreamResult { + shared_context: &Arc, + ) -> StreamResult { // The "stateful" here means that the executor may issue read operations to the state store // massively and continuously. Used to decide whether to apply the optimization of subtasks. fn is_stateful_executor(stream_node: &StreamNode) -> bool { @@ -504,6 +431,7 @@ impl LocalStreamManagerCore { vnode_bitmap.clone(), has_stateful || is_stateful, subtasks, + shared_context, ) .await?, ); @@ -523,21 +451,22 @@ impl LocalStreamManagerCore { let schema: Schema = node.fields.iter().map(Field::from).collect(); let identity = format!("{} {:X}", node.get_node_body().unwrap(), executor_id); + let info = ExecutorInfo { + schema, + pk_indices, + identity, + }; + let eval_error_report = ActorEvalErrorReport { actor_context: actor_context.clone(), - identity: identity.clone().into(), + identity: info.identity.clone().into(), }; // Build the executor with params. let executor_params = ExecutorParams { env: env.clone(), - info: ExecutorInfo { - schema: schema.clone(), - pk_indices: pk_indices.clone(), - identity: identity.clone(), - }, - + info: info.clone(), executor_id, operator_id, op_info, @@ -547,29 +476,20 @@ impl LocalStreamManagerCore { actor_context: actor_context.clone(), vnode_bitmap, eval_error_report, + watermark_epoch: self.watermark_epoch.clone(), + shared_context: shared_context.clone(), + local_barrier_manager: shared_context.local_barrier_manager.clone(), }; - let executor = create_executor(executor_params, self, node, store).await?; - assert_eq!( - executor.pk_indices(), - &pk_indices, - "`pk_indices` of {} not consistent with what derived by optimizer", - executor.identity() - ); - assert_eq!( - executor.schema(), - &schema, - "`schema` of {} not consistent with what derived by optimizer", - executor.identity() - ); + let executor = create_executor(executor_params, node, store).await?; // Wrap the executor for debug purpose. - let executor = WrapperExecutor::new( + let wrapped = WrapperExecutor::new( executor, actor_context.clone(), - self.config.developer.enable_executor_row_count, - ) - .boxed(); + env.config().developer.enable_executor_row_count, + ); + let executor = (info, wrapped).into(); // If there're multiple stateful executors in this actor, we will wrap it into a subtask. let executor = if has_stateful && is_stateful { @@ -589,16 +509,17 @@ impl LocalStreamManagerCore { /// Create a chain(tree) of nodes and return the head executor. async fn create_nodes( - &mut self, + &self, fragment_id: FragmentId, node: &stream_plan::StreamNode, env: StreamEnvironment, actor_context: &ActorContextRef, vnode_bitmap: Option, - ) -> StreamResult<(BoxedExecutor, Vec)> { + shared_context: &Arc, + ) -> StreamResult<(Executor, Vec)> { let mut subtasks = vec![]; - let executor = dispatch_state_store!(self.state_store.clone(), store, { + let executor = dispatch_state_store!(env.state_store(), store, { self.create_nodes_inner( fragment_id, node, @@ -608,6 +529,7 @@ impl LocalStreamManagerCore { vnode_bitmap, false, &mut subtasks, + shared_context, ) .await })?; @@ -615,23 +537,20 @@ impl LocalStreamManagerCore { Ok((executor, subtasks)) } - async fn build_actors( - &mut self, - actors: &[ActorId], - env: StreamEnvironment, - ) -> StreamResult<()> { - for &actor_id in actors { - let actor = self - .actors - .remove(&actor_id) - .ok_or_else(|| anyhow!("No such actor with actor id:{}", actor_id))?; - let mview_definition = &actor.mview_definition; - let actor_context = ActorContext::create_with_metrics( - actor_id, - actor.fragment_id, - self.total_mem_val.clone(), + async fn create_actors( + self: Arc, + actors: Vec, + shared_context: Arc, + ) -> StreamResult>> { + let mut ret = Vec::with_capacity(actors.len()); + for actor in actors { + let actor_id = actor.actor_id; + let actor_context = ActorContext::create( + &actor, + self.env.total_mem_usage(), self.streaming_metrics.clone(), - self.config.unique_user_stream_errors, + self.env.config().unique_user_stream_errors, + actor.dispatcher.len(), ); let vnode_bitmap = actor.vnode_bitmap.as_ref().map(|b| b.into()); let expr_context = actor.expr_context.clone().unwrap(); @@ -640,43 +559,59 @@ impl LocalStreamManagerCore { .create_nodes( actor.fragment_id, actor.get_nodes()?, - env.clone(), + self.env.clone(), &actor_context, vnode_bitmap, + &shared_context, ) // If hummock tracing is not enabled, it directly returns wrapped future. .may_trace_hummock() .await?; - let dispatcher = - self.create_dispatcher(executor, &actor.dispatcher, actor_id, actor.fragment_id)?; + let dispatcher = self.create_dispatcher( + executor, + &actor.dispatcher, + actor_id, + actor.fragment_id, + &shared_context, + )?; let actor = Actor::new( dispatcher, subtasks, - self.context.clone(), self.streaming_metrics.clone(), actor_context.clone(), expr_context, + shared_context.local_barrier_manager.clone(), ); + ret.push(actor); + } + Ok(ret) + } +} + +impl LocalBarrierWorker { + pub(super) fn spawn_actors(&mut self, actors: Vec>) { + for actor in actors { let monitor = tokio_metrics::TaskMonitor::new(); + let actor_context = actor.actor_context.clone(); + let actor_id = actor_context.id; let handle = { - let context = self.context.clone(); - let actor = async move { - if let Err(err) = actor.run().await { + let trace_span = format!("Actor {actor_id}: `{}`", actor_context.mview_definition); + let barrier_manager = self.current_shared_context.local_barrier_manager.clone(); + let actor = actor.run().map(move |result| { + if let Err(err) = result { // TODO: check error type and panic if it's unexpected. // Intentionally use `?` on the report to also include the backtrace. tracing::error!(actor_id, error = ?err.as_report(), "actor exit with error"); - context.barrier_manager().notify_failure(actor_id, err); + barrier_manager.notify_failure(actor_id, err); } - }; - let traced = match &mut self.await_tree_reg { + }); + let traced = match &self.actor_manager.await_tree_reg { Some(m) => m - .register( - actor_id, - format!("Actor {actor_id}: `{}`", mview_definition), - ) + .lock() + .register(actor_id, trace_span) .instrument(actor) .left_future(), None => actor.right_future(), @@ -684,7 +619,7 @@ impl LocalStreamManagerCore { let instrumented = monitor.instrument(traced); #[cfg(enable_task_local_alloc)] { - let metrics = self.streaming_metrics.clone(); + let metrics = streaming_metrics.clone(); let actor_id_str = actor_id.to_string(); let fragment_id_str = actor_context.fragment_id.to_string(); let allocation_stated = task_stats_alloc::allocation_stat( @@ -703,16 +638,16 @@ impl LocalStreamManagerCore { } #[cfg(not(enable_task_local_alloc))] { - self.runtime.spawn(instrumented) + self.actor_manager.runtime.spawn(instrumented) } }; - self.handles.insert(actor_id, handle); + self.actor_manager_state.handles.insert(actor_id, handle); - if self.streaming_metrics.level >= MetricLevel::Debug { + if self.actor_manager.streaming_metrics.level >= MetricLevel::Debug { tracing::info!("Tokio metrics are enabled because metrics_level >= Debug"); let actor_id_str = actor_id.to_string(); - let metrics = self.streaming_metrics.clone(); - let actor_monitor_task = self.runtime.spawn(async move { + let metrics = self.actor_manager.streaming_metrics.clone(); + let actor_monitor_task = self.actor_manager.runtime.spawn(async move { loop { let task_metrics = monitor.cumulative(); metrics @@ -762,56 +697,41 @@ impl LocalStreamManagerCore { tokio::time::sleep(Duration::from_secs(1)).await; } }); - self.actor_monitor_tasks + self.actor_manager_state + .actor_monitor_tasks .insert(actor_id, actor_monitor_task); } } - - Ok(()) - } - - pub fn take_all_handles(&mut self) -> StreamResult> { - Ok(std::mem::take(&mut self.handles)) - } - - pub fn remove_actor_handles( - &mut self, - actor_ids: &[ActorId], - ) -> StreamResult> { - actor_ids - .iter() - .map(|actor_id| { - self.handles - .remove(actor_id) - .ok_or_else(|| anyhow!("No such actor with actor id:{}", actor_id).into()) - }) - .try_collect() } +} - fn update_actor_info(&mut self, new_actor_infos: &[ActorInfo]) -> StreamResult<()> { - let mut actor_infos = self.context.actor_infos.write(); +impl LocalBarrierWorker { + /// This function could only be called once during the lifecycle of `LocalStreamManager` for + /// now. + pub fn update_actor_info(&self, new_actor_infos: Vec) -> StreamResult<()> { + let mut actor_infos = self.current_shared_context.actor_infos.write(); for actor in new_actor_infos { - let ret = actor_infos.insert(actor.get_actor_id(), actor.clone()); - if let Some(prev_actor) = ret - && actor != &prev_actor + if let Some(prev_actor) = actor_infos.get(&actor.get_actor_id()) + && &actor != prev_actor { bail!( "actor info mismatch when broadcasting {}", actor.get_actor_id() ); } + actor_infos.insert(actor.get_actor_id(), actor); } Ok(()) } +} +impl StreamActorManagerState { /// `drop_actor` is invoked by meta node via RPC once the stop barrier arrives at the /// sink. All the actors in the actors should stop themselves before this method is invoked. fn drop_actor(&mut self, actor_id: ActorId) { - self.context.retain_channel(|&(up_id, _)| up_id != actor_id); self.actor_monitor_tasks .remove(&actor_id) .inspect(|handle| handle.abort()); - self.context.actor_infos.write().remove(&actor_id); self.actors.remove(&actor_id); // Task should have already stopped when this method is invoked. There might be some @@ -820,41 +740,27 @@ impl LocalStreamManagerCore { self.handles.remove(&actor_id); } + fn drain_actor_handles(&mut self) -> Vec<(ActorId, ActorHandle)> { + self.handles.drain().collect() + } + /// `stop_all_actors` is invoked by meta node via RPC for recovery purpose. Different from the /// `drop_actor`, the execution of the actors will be aborted. - async fn stop_all_actors(&mut self) { - for (actor_id, handle) in &self.handles { - tracing::debug!("force stopping actor {}", actor_id); - handle.abort(); - } - for (actor_id, handle) in self.handles.drain() { - tracing::debug!("join actor {}", actor_id); - let result = handle.await; - assert!(result.is_ok() || result.unwrap_err().is_cancelled()); - } + fn clear_state(&mut self) { self.actors.clear(); - self.context.clear_channels(); - if let Some(m) = self.await_tree_reg.as_mut() { - m.clear(); - } self.actor_monitor_tasks.clear(); - self.context.actor_infos.write().clear(); } - fn update_actors(&mut self, actors: &[stream_plan::StreamActor]) -> StreamResult<()> { + fn update_actors(&mut self, actors: Vec) -> StreamResult<()> { for actor in actors { + let actor_id = actor.get_actor_id(); self.actors - .try_insert(actor.get_actor_id(), actor.clone()) - .map_err(|_| anyhow!("duplicated actor {}", actor.get_actor_id()))?; + .try_insert(actor_id, actor) + .map_err(|_| anyhow!("duplicated actor {}", actor_id))?; } Ok(()) } - - /// When executor need to create cache, it will call this needs the watermark epoch for evict. - pub fn get_watermark_epoch(&self) -> AtomicU64Ref { - self.watermark_epoch.clone() - } } #[cfg(test)] diff --git a/src/stream/tests/integration_tests/eowc_over_window.rs b/src/stream/tests/integration_tests/eowc_over_window.rs index b085cccb76dc1..cc674e556ab5e 100644 --- a/src/stream/tests/integration_tests/eowc_over_window.rs +++ b/src/stream/tests/integration_tests/eowc_over_window.rs @@ -14,9 +14,7 @@ use risingwave_expr::aggregate::{AggArgs, AggKind}; use risingwave_expr::window_function::{Frame, FrameBound, WindowFuncCall, WindowFuncKind}; -use risingwave_stream::executor::{ - EowcOverWindowExecutor, EowcOverWindowExecutorArgs, ExecutorInfo, -}; +use risingwave_stream::executor::{EowcOverWindowExecutor, EowcOverWindowExecutorArgs}; use crate::prelude::*; @@ -54,7 +52,6 @@ async fn create_executor( }); Schema { fields } }; - let output_pk_indices = vec![2]; let state_table = StateTable::new_without_distribution_inconsistent_op( store, @@ -65,17 +62,14 @@ async fn create_executor( ) .await; - let (tx, source) = MockSource::channel(input_schema, input_pk_indices.clone()); + let (tx, source) = MockSource::channel(); + let source = source.into_executor(input_schema, input_pk_indices.clone()); let executor = EowcOverWindowExecutor::new(EowcOverWindowExecutorArgs { - actor_ctx: ActorContext::create(123), - info: ExecutorInfo { - schema: output_schema, - pk_indices: output_pk_indices, - identity: "EowcOverWindowExecutor".to_string(), - }, + actor_ctx: ActorContext::for_test(123), - input: source.boxed(), + input: source, + schema: output_schema, calls, partition_key_indices, order_key_index, diff --git a/src/stream/tests/integration_tests/hash_agg.rs b/src/stream/tests/integration_tests/hash_agg.rs index 7704746b28e36..10e297ec634e4 100644 --- a/src/stream/tests/integration_tests/hash_agg.rs +++ b/src/stream/tests/integration_tests/hash_agg.rs @@ -39,10 +39,11 @@ async fn test_hash_agg_count_sum() { AggCall::from_pretty("(sum:int8 $2:int8)"), ]; - let (mut tx, source) = MockSource::channel(schema, PkIndices::new()); + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, PkIndices::new()); let hash_agg = new_boxed_hash_agg_executor( store, - Box::new(source), + source, false, agg_calls, 0, @@ -117,10 +118,11 @@ async fn test_hash_agg_min() { AggCall::from_pretty("(min:int8 $1:int8)"), ]; - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![2]); let hash_agg = new_boxed_hash_agg_executor( store, - Box::new(source), + source, false, agg_calls, 0, @@ -192,10 +194,11 @@ async fn test_hash_agg_min_append_only() { AggCall::from_pretty("(min:int8 $1:int8)"), ]; - let (mut tx, source) = MockSource::channel(schema, vec![2]); // pk + let (mut tx, source) = MockSource::channel(); + let source = source.into_executor(schema, vec![2]); let hash_agg = new_boxed_hash_agg_executor( store, - Box::new(source), + source, true, // is append only agg_calls, 0, @@ -267,10 +270,11 @@ async fn test_hash_agg_emit_on_window_close() { let agg_calls = vec![AggCall::from_pretty("(count:int8)")]; let create_executor = || async { - let (tx, source) = MockSource::channel(input_schema.clone(), PkIndices::new()); + let (tx, source) = MockSource::channel(); + let source = source.into_executor(input_schema.clone(), PkIndices::new()); let hash_agg = new_boxed_hash_agg_executor( store.clone(), - Box::new(source), + source, false, agg_calls.clone(), 0, diff --git a/src/stream/tests/integration_tests/hop_window.rs b/src/stream/tests/integration_tests/hop_window.rs index 6f60b3bd1e9fd..97e2886dbecda 100644 --- a/src/stream/tests/integration_tests/hop_window.rs +++ b/src/stream/tests/integration_tests/hop_window.rs @@ -16,7 +16,7 @@ use risingwave_common::types::test_utils::IntervalTestExt; use risingwave_common::types::{Interval, Timestamp}; use risingwave_expr::expr::test_utils::make_hop_window_expression; use risingwave_expr::expr::NonStrictExpression; -use risingwave_stream::executor::{ExecutorInfo, HopWindowExecutor}; +use risingwave_stream::executor::HopWindowExecutor; use crate::prelude::*; @@ -24,12 +24,15 @@ const TIME_COL_IDX: usize = 2; const CHUNK_SIZE: usize = 256; fn create_executor(output_indices: Vec) -> (MessageSender, BoxedMessageStream) { - let field1 = Field::unnamed(DataType::Int64); - let field2 = Field::unnamed(DataType::Int64); - let field3 = Field::with_name(DataType::Timestamp, "created_at"); - let schema = Schema::new(vec![field1, field2, field3]); - let pk_indices = vec![0]; - let (tx, source) = MockSource::channel(schema.clone(), pk_indices.clone()); + let (tx, source) = MockSource::channel(); + let source = source.into_executor( + Schema::new(vec![ + Field::unnamed(DataType::Int64), + Field::unnamed(DataType::Int64), + Field::with_name(DataType::Timestamp, "created_at"), + ]), + vec![0], + ); let window_slide = Interval::from_minutes(15); let window_size = Interval::from_minutes(30); @@ -46,13 +49,8 @@ fn create_executor(output_indices: Vec) -> (MessageSender, BoxedMessageSt ( tx, HopWindowExecutor::new( - ActorContext::create(123), - ExecutorInfo { - schema, - pk_indices, - identity: "HopWindowExecutor".to_string(), - }, - Box::new(source), + ActorContext::for_test(123), + source, TIME_COL_IDX, window_slide, window_size, diff --git a/src/stream/tests/integration_tests/over_window.rs b/src/stream/tests/integration_tests/over_window.rs index 1b7ee0e304899..0be8e1848e9cd 100644 --- a/src/stream/tests/integration_tests/over_window.rs +++ b/src/stream/tests/integration_tests/over_window.rs @@ -18,7 +18,7 @@ use risingwave_expr::window_function::{ Frame, FrameBound, FrameExclusion, WindowFuncCall, WindowFuncKind, }; use risingwave_stream::executor::monitor::StreamingMetrics; -use risingwave_stream::executor::{ExecutorInfo, OverWindowExecutor, OverWindowExecutorArgs}; +use risingwave_stream::executor::{OverWindowExecutor, OverWindowExecutorArgs}; use crate::prelude::*; @@ -63,7 +63,6 @@ async fn create_executor( }); Schema { fields } }; - let output_pk_indices = vec![2]; let state_table = StateTable::new_without_distribution( store, @@ -74,17 +73,14 @@ async fn create_executor( ) .await; - let (tx, source) = MockSource::channel(input_schema, input_pk_indices.clone()); + let (tx, source) = MockSource::channel(); + let source = source.into_executor(input_schema, input_pk_indices.clone()); let executor = OverWindowExecutor::new(OverWindowExecutorArgs { - actor_ctx: ActorContext::create(123), - info: ExecutorInfo { - schema: output_schema, - pk_indices: output_pk_indices, - identity: "OverWindowExecutor".to_string(), - }, + actor_ctx: ActorContext::for_test(123), - input: source.boxed(), + input: source, + schema: output_schema, calls, partition_key_indices, order_key_indices, diff --git a/src/stream/tests/integration_tests/project_set.rs b/src/stream/tests/integration_tests/project_set.rs index 8c05bc43aa694..5cff03284f4b6 100644 --- a/src/stream/tests/integration_tests/project_set.rs +++ b/src/stream/tests/integration_tests/project_set.rs @@ -14,7 +14,7 @@ use multimap::MultiMap; use risingwave_expr::table_function::repeat; -use risingwave_stream::executor::{ExecutorInfo, ProjectSetExecutor}; +use risingwave_stream::executor::ProjectSetExecutor; use crate::prelude::*; @@ -27,30 +27,17 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) { Field::unnamed(DataType::Int64), ], }; - let (tx, source) = MockSource::channel(schema, PkIndices::new()); + let (tx, source) = MockSource::channel(); + let source = source.into_executor(schema, PkIndices::new()); let test_expr = build_from_pretty("(add:int8 $0:int8 $1:int8)").into_inner(); let test_expr_watermark = build_from_pretty("(add:int8 $0:int8 1:int8)").into_inner(); let tf1 = repeat(build_from_pretty("1:int4").into_inner(), 1); let tf2 = repeat(build_from_pretty("2:int4").into_inner(), 2); - let info = ExecutorInfo { - schema: Schema { - fields: vec![ - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int64), - Field::unnamed(DataType::Int32), - Field::unnamed(DataType::Int32), - ], - }, - pk_indices: vec![], - identity: "ProjectSetExecutor".to_string(), - }; - - let project_set = Box::new(ProjectSetExecutor::new( - ActorContext::create(123), - info, - Box::new(source), + let project_set = ProjectSetExecutor::new( + ActorContext::for_test(123), + source, vec![ test_expr.into(), test_expr_watermark.into(), @@ -60,8 +47,8 @@ fn create_executor() -> (MessageSender, BoxedMessageStream) { CHUNK_SIZE, MultiMap::from_iter(std::iter::once((0, 1))), vec![], - )); - (tx, project_set.execute()) + ); + (tx, project_set.boxed().execute()) } #[tokio::test] diff --git a/src/test_runner/src/test_runner.rs b/src/test_runner/src/test_runner.rs index 38481880b25f0..14db70d351186 100644 --- a/src/test_runner/src/test_runner.rs +++ b/src/test_runner/src/test_runner.rs @@ -82,7 +82,7 @@ pub fn run_test_inner(cases: &[&TestDescAndFn], hook: impl TestHook + 'static + test_main(&args, cases, None) } -thread_local!(static FS: RefCell>> = RefCell::new(None)); +thread_local!(static FS: RefCell>> = const { RefCell::new(None) }); #[derive(Clone)] struct FailPointHook; diff --git a/src/tests/compaction_test/Makefile.toml b/src/tests/compaction_test/Makefile.toml index 2b1c0405edbfd..2ca5332975c17 100644 --- a/src/tests/compaction_test/Makefile.toml +++ b/src/tests/compaction_test/Makefile.toml @@ -7,12 +7,3 @@ script = """ set -e cargo run --bin compaction-test --profile "${RISINGWAVE_BUILD_PROFILE}" -- "$@" """ - -[tasks.delete-range-test] -category = "RiseDev - Test" -description = "Run hummock compaction delete-range for test" -script = """ -#!/usr/bin/env bash -set -e -cargo run --bin delete-range-test --profile "${RISINGWAVE_BUILD_PROFILE}" -- "$@" -""" diff --git a/src/tests/compaction_test/src/bin/delete_range.rs b/src/tests/compaction_test/src/bin/delete_range.rs index 5950a4111f71e..f154314fb5c8a 100644 --- a/src/tests/compaction_test/src/bin/delete_range.rs +++ b/src/tests/compaction_test/src/bin/delete_range.rs @@ -16,6 +16,7 @@ #[cfg_attr(coverage, coverage(off))] fn main() { + // Since we decide to record watermark in every state-table to replace delete-range, this test is not need again. We keep it because we may need delete-range in some day for other features. use clap::Parser; let opts = risingwave_compaction_test::CompactionTestOpts::parse(); diff --git a/src/tests/compaction_test/src/delete_range_runner.rs b/src/tests/compaction_test/src/delete_range_runner.rs index 1c5f80ee6cf35..f2add3f4c289f 100644 --- a/src/tests/compaction_test/src/delete_range_runner.rs +++ b/src/tests/compaction_test/src/delete_range_runner.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::collections::HashMap; use std::future::Future; use std::ops::{Bound, RangeBounds}; use std::pin::{pin, Pin}; @@ -25,11 +24,11 @@ use futures::StreamExt; use rand::rngs::StdRng; use rand::{RngCore, SeedableRng}; use risingwave_common::cache::CachePriority; -use risingwave_common::catalog::hummock::PROPERTIES_RETENTION_SECOND_KEY; use risingwave_common::catalog::TableId; use risingwave_common::config::{ extract_storage_memory_config, load_config, NoOverride, ObjectStoreConfig, RwConfig, }; +use risingwave_common::system_param::reader::SystemParamsRead; use risingwave_common::util::epoch::{test_epoch, EpochExt, EPOCH_INC_MIN_STEP_FOR_TEST}; use risingwave_hummock_sdk::compaction_group::StaticCompactionGroupId; use risingwave_hummock_sdk::key::TableKey; @@ -55,6 +54,7 @@ use risingwave_storage::hummock::sstable_store::SstableStoreRef; use risingwave_storage::hummock::utils::cmp_delete_range_left_bounds; use risingwave_storage::hummock::{ CachePolicy, FileCache, HummockStorage, MemoryLimiter, SstableObjectIdManager, SstableStore, + SstableStoreConfig, }; use risingwave_storage::monitor::{CompactorMetrics, HummockStateStoreMetrics}; use risingwave_storage::opts::StorageOpts; @@ -134,10 +134,7 @@ async fn compaction_test( distribution_key: vec![], stream_key: vec![], owner: 0, - properties: HashMap::::from([( - PROPERTIES_RETENTION_SECOND_KEY.to_string(), - 0.to_string(), - )]), + retention_seconds: None, fragment_id: 0, dml_fragment_id: None, initialized_at_epoch: None, @@ -212,18 +209,19 @@ async fn compaction_test( ObjectStoreConfig::default(), ) .await; - let sstable_store = Arc::new(SstableStore::new( - Arc::new(remote_object_store), - system_params.data_directory().to_string(), - storage_memory_config.block_cache_capacity_mb * (1 << 20), - storage_memory_config.meta_cache_capacity_mb * (1 << 20), - 0, - storage_memory_config.prefetch_buffer_capacity_mb * (1 << 20), - storage_opts.max_prefetch_block_number, - FileCache::none(), - FileCache::none(), - None, - )); + let sstable_store = Arc::new(SstableStore::new(SstableStoreConfig { + store: Arc::new(remote_object_store), + path: system_params.data_directory().to_string(), + block_cache_capacity: storage_memory_config.block_cache_capacity_mb * (1 << 20), + meta_cache_capacity: storage_memory_config.meta_cache_capacity_mb * (1 << 20), + high_priority_ratio: 0, + prefetch_buffer_capacity: storage_memory_config.prefetch_buffer_capacity_mb * (1 << 20), + max_prefetch_block_number: storage_opts.max_prefetch_block_number, + data_file_cache: FileCache::none(), + meta_file_cache: FileCache::none(), + recent_filter: None, + state_store_metrics: state_store_metrics.clone(), + })); let store = HummockStorage::new( storage_opts.clone(), @@ -422,13 +420,14 @@ impl NormalState { async fn commit_impl( &mut self, - delete_ranges: Vec<(Bound, Bound)>, + _delete_ranges: Vec<(Bound, Bound)>, next_epoch: u64, ) -> Result<(), String> { - self.storage - .flush(delete_ranges) - .await - .map_err(|e| format!("{:?}", e))?; + // self.storage + // .flush(delete_ranges) + // .await + // .map_err(|e| format!("{:?}", e))?; + self.storage.flush().await.map_err(|e| format!("{:?}", e))?; self.storage .seal_current_epoch(next_epoch, SealCurrentEpochOptions::for_test()); Ok(()) @@ -630,6 +629,8 @@ mod tests { use super::compaction_test; + #[ignore] + // TODO: may modify the test to use per vnode table watermark #[tokio::test(flavor = "multi_thread", worker_threads = 3)] async fn test_small_data() { let config = RwConfig::default(); diff --git a/src/tests/e2e_extended_mode/Cargo.toml b/src/tests/e2e_extended_mode/Cargo.toml index 56128c9b45f39..81644ee3430cf 100644 --- a/src/tests/e2e_extended_mode/Cargo.toml +++ b/src/tests/e2e_extended_mode/Cargo.toml @@ -18,7 +18,7 @@ anyhow = { version = "1", features = ["backtrace"] } chrono = { version = "0.4", features = ['serde'] } clap = { version = "4", features = ["derive"] } pg_interval = "0.4" -rust_decimal ={ version = "1.33", features = ["db-postgres"] } +rust_decimal ={ version = "1.34", features = ["db-postgres"] } tokio = { version = "0.2.24", package = "madsim-tokio", features = ["rt", "macros","rt-multi-thread"] } tokio-postgres = { version = "0.7", features = ["with-chrono-0_4"] } tracing = "0.1" diff --git a/src/tests/e2e_extended_mode/src/test.rs b/src/tests/e2e_extended_mode/src/test.rs index 3e822bbd04d31..26c9b9d4c2c96 100644 --- a/src/tests/e2e_extended_mode/src/test.rs +++ b/src/tests/e2e_extended_mode/src/test.rs @@ -75,6 +75,7 @@ impl TestSuite { self.simple_cancel(true).await?; self.complex_cancel(false).await?; self.complex_cancel(true).await?; + self.subquery_with_param().await?; Ok(()) } @@ -541,4 +542,17 @@ impl TestSuite { new_client.execute("drop table t3", &[]).await?; Ok(()) } + + async fn subquery_with_param(&self) -> anyhow::Result<()> { + let client = self.create_client(false).await?; + + let res = client + .query("select (select $1::SMALLINT)", &[&1024_i16]) + .await + .unwrap(); + + assert_eq!(res[0].get::(0), 1024_i16); + + Ok(()) + } } diff --git a/src/tests/libpq_test/Cargo.lock b/src/tests/libpq_test/Cargo.lock index 74beef901a9ea..686143d827141 100644 --- a/src/tests/libpq_test/Cargo.lock +++ b/src/tests/libpq_test/Cargo.lock @@ -4,67 +4,66 @@ version = 3 [[package]] name = "aho-corasick" -version = "1.0.2" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43f6cb1bf222025340178f382c426f13757b2960e89779dfcb319c32542a5a41" +checksum = "b2969dcb958b36655471fc61f7e416fa76033bdd4bfed0678d8fee1e2d07a1f0" dependencies = [ "memchr", ] [[package]] name = "anstream" -version = "0.3.2" +version = "0.6.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca84f3628370c59db74ee214b3263d58f9aadd9b4fe7e711fd87dc452b7f163" +checksum = "d96bd03f33fe50a863e394ee9718a706f988b9079b20c3784fb726e7678b62fb" dependencies = [ "anstyle", "anstyle-parse", "anstyle-query", "anstyle-wincon", "colorchoice", - "is-terminal", "utf8parse", ] [[package]] name = "anstyle" -version = "1.0.1" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a30da5c5f2d5e72842e00bcb57657162cdabef0931f40e2deb9b4140440cecd" +checksum = "8901269c6307e8d93993578286ac0edf7f195079ffff5ebdeea6a59ffb7e36bc" [[package]] name = "anstyle-parse" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +checksum = "c75ac65da39e5fe5ab759307499ddad880d724eed2f6ce5b5e8a26f4f387928c" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.0.0" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +checksum = "e28923312444cdd728e4738b3f9c9cac739500909bb3d3c94b43551b16517648" dependencies = [ - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anstyle-wincon" -version = "1.0.1" +version = "3.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "180abfa45703aebe0093f79badacc01b8fd4ea2e35118747e5811127f926e188" +checksum = "1cd54b81ec8d6180e24654d0b371ad22fc3dd083b6ff8ba325b72e00c87660a7" dependencies = [ "anstyle", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "anyhow" -version = "1.0.71" +version = "1.0.80" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c7d0618f0e0b7e8ff11427422b64564d5fb0be1940354bfe2e0529b18a9d9b8" +checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1" [[package]] name = "bindgen" @@ -96,15 +95,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.4.1" +version = "2.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "327762f6e5a765692301e5bb513e0d9fef63be86bbc14528052b1cd3e6f03e07" - -[[package]] -name = "cc" -version = "1.0.79" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50d30906286121d95be3d479533b458f87493b30a4b5f79a607db8f5d11aa91f" +checksum = "ed570934406eb16438a4e976b1b4500774099c13b8cb96eec99f620f05090ddf" [[package]] name = "cexpr" @@ -123,9 +116,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "clang-sys" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f" +checksum = "67523a3b4be3ce1989d607a828d036249522dd9c1c8de7f4dd2dae43a37369d1" dependencies = [ "glob", "libc", @@ -134,20 +127,19 @@ dependencies = [ [[package]] name = "clap" -version = "4.3.11" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1640e5cc7fb47dbb8338fd471b105e7ed6c3cb2aeb00c2e067127ffd3764a05d" +checksum = "c918d541ef2913577a0f9566e9ce27cb35b6df072075769e0b26cb5a554520da" dependencies = [ "clap_builder", "clap_derive", - "once_cell", ] [[package]] name = "clap_builder" -version = "4.3.11" +version = "4.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "98c59138d527eeaf9b53f35a77fcc1fad9d883116070c63d5de1c7dc7b00c72b" +checksum = "9f3e7391dad68afb0c2ede1bf619f579a3dc9c2ec67f089baa397123a2f3d1eb" dependencies = [ "anstream", "anstyle", @@ -157,21 +149,21 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.3.2" +version = "4.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8cd2b2a819ad6eec39e8f1d6b53001af1e5469f8c177579cdaeb313115b825f" +checksum = "307bc0538d5f0f83b8248db3087aa92fe504e4691294d0c96c0eabc33f47ba47" dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.51", ] [[package]] name = "clap_lex" -version = "0.5.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2da6da31387c7e4ef160ffab6d5e7f00c42626fe39aea70a7b0f1773f7dd6c1b" +checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce" [[package]] name = "colorchoice" @@ -181,29 +173,18 @@ checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" [[package]] name = "either" -version = "1.8.1" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcaabb2fef8c910e7f4c7ce9f67a1283a1715879a7c230ca9d6d1ae31f16d91" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" [[package]] name = "errno" -version = "0.3.1" +version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" +checksum = "a258e46cdc063eb8519c00b9fc845fc47bcfca4130e2f08e88665ceda8474245" dependencies = [ - "errno-dragonfly", - "libc", - "windows-sys", -] - -[[package]] -name = "errno-dragonfly" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa68f1b12764fab894d2755d2518754e71b4fd80ecfb822714a1206c2aab39bf" -dependencies = [ - "cc", "libc", + "windows-sys 0.52.0", ] [[package]] @@ -219,20 +200,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" [[package]] -name = "hermit-abi" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" - -[[package]] -name = "is-terminal" -version = "0.4.9" +name = "home" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb0889898416213fab133e1d33a0e5858a48177452750691bde3666d0fdbaf8b" +checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" dependencies = [ - "hermit-abi", - "rustix", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] @@ -249,18 +222,18 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "libc" -version = "0.2.149" +version = "0.2.153" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a08173bc88b7955d1b3145aa561539096c421ac8debde8cbc3612ec635fee29b" +checksum = "9c198f91728a82281a64e1f4f9eeb25d82cb32a5de251c6bd1b5154d63a8e7bd" [[package]] name = "libloading" -version = "0.7.4" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b67380fd3b2fbe7527a606e18729d21c6f3951633d0500574c4dc22d2d638b9f" +checksum = "c571b676ddfc9a8c12f1f3d3085a7b163966a8fd8098a90640953ce5f6170161" dependencies = [ "cfg-if", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -288,21 +261,21 @@ dependencies = [ [[package]] name = "linux-raw-sys" -version = "0.4.10" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da2479e8c062e40bf0066ffa0bc823de0a9368974af99c9f6df941d2c231e03f" +checksum = "01cda141df6706de531b6c46c3a33ecca755538219bd484262fa09410c13539c" [[package]] name = "log" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b06a4cde4c0f271a446782e3eff8de789548ce57dbc8eca9292c27f4a42004b4" +checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.5.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" [[package]] name = "minimal-lexical" @@ -322,9 +295,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] name = "peeking_take_while" @@ -334,33 +307,33 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" [[package]] name = "pkg-config" -version = "0.3.27" +version = "0.3.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26072860ba924cbfa98ea39c8c19b4dd6a4a25423dbdf219c1eca91aa0cf6964" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" [[package]] name = "proc-macro2" -version = "1.0.64" +version = "1.0.78" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78803b62cbf1f46fde80d7c0e803111524b9877184cfe7c3033659490ac7a7da" +checksum = "e2422ad645d89c99f8f3e6b88a9fdeca7fabeac836b1002371c4367c8f984aae" dependencies = [ "unicode-ident", ] [[package]] name = "quote" -version = "1.0.29" +version = "1.0.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "573015e8ab27661678357f27dc26460738fd2b6c86e46f386fde94cb5d913105" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" dependencies = [ "proc-macro2", ] [[package]] name = "regex" -version = "1.9.1" +version = "1.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2eae68fc220f7cf2532e4494aded17545fce192d59cd996e0fe7887f4ceb575" +checksum = "b62dbe01f0b06f9d8dc7d49e05a0785f153b00b2c227856282f671e0318c9b15" dependencies = [ "aho-corasick", "memchr", @@ -370,9 +343,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39354c10dd07468c2e73926b23bb9c2caca74c5501e38a35da70406f1d923310" +checksum = "5bb987efffd3c6d0d8f5f89510bb458559eab11e4f869acb20bf845e016259cd" dependencies = [ "aho-corasick", "memchr", @@ -381,9 +354,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "risingwave_libpq_test" @@ -402,28 +375,28 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "6ea3e1a662af26cd7a3ba09c0297a31af215563ecf42817c98df621387f4e949" dependencies = [ - "bitflags 2.4.1", + "bitflags 2.4.2", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.52.0", ] [[package]] name = "shlex" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "strsim" -version = "0.10.0" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +checksum = "5ee073c9e4cd00e28217186dbe12796d692868f432bf2e97ee73bed0c56dfa01" [[package]] name = "syn" @@ -438,9 +411,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.25" +version = "2.0.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15e3fc8c0c74267e2df136e5e5fb656a464158aa57624053375eb9c8c6e25ae2" +checksum = "6ab617d94515e94ae53b8406c628598680aa0c9587474ecbe58188f7b345d66c" dependencies = [ "proc-macro2", "quote", @@ -449,29 +422,29 @@ dependencies = [ [[package]] name = "thiserror" -version = "1.0.43" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a35fc5b8971143ca348fa6df4f024d4d55264f3468c71ad1c2f365b0a4d58c42" +checksum = "1e45bcbe8ed29775f228095caf2cd67af7a4ccf756ebff23a306bf3e8b47b24b" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.43" +version = "1.0.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463fe12d7993d3b327787537ce8dd4dfa058de32fc2b195ef3cde03dc4771e8f" +checksum = "a953cb265bef375dae3de6663da4d3804eee9682ea80d8e2542529b73c531c81" dependencies = [ "proc-macro2", "quote", - "syn 2.0.25", + "syn 2.0.51", ] [[package]] name = "unicode-ident" -version = "1.0.10" +version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22049a19f4a68748a168c0fc439f9516686aa045927ff767eca0a85101fb6e73" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "utf8parse" @@ -487,99 +460,144 @@ checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "which" -version = "4.4.0" +version = "4.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2441c784c52b289a054b7201fc93253e288f094e2f4be9058343127c4226a269" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" dependencies = [ "either", - "libc", + "home", "once_cell", + "rustix", ] [[package]] -name = "winapi" -version = "0.3.9" +name = "windows-sys" +version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ - "winapi-i686-pc-windows-gnu", - "winapi-x86_64-pc-windows-gnu", + "windows-targets 0.48.5", ] [[package]] -name = "winapi-i686-pc-windows-gnu" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" - -[[package]] -name = "winapi-x86_64-pc-windows-gnu" -version = "0.4.0" +name = "windows-sys" +version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.3", +] [[package]] -name = "windows-sys" -version = "0.48.0" +name = "windows-targets" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ - "windows-targets", + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", ] [[package]] name = "windows-targets" -version = "0.48.1" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05d4b17490f70499f20b9e791dcf6a299785ce8af4d709018206dc5b4953e95f" +checksum = "d380ba1dc7187569a8a9e91ed34b8ccfc33123bbacb8c0aed2d1ad7f3ef2dc5f" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.52.3", + "windows_aarch64_msvc 0.52.3", + "windows_i686_gnu 0.52.3", + "windows_i686_msvc 0.52.3", + "windows_x86_64_gnu 0.52.3", + "windows_x86_64_gnullvm 0.52.3", + "windows_x86_64_msvc 0.52.3", ] [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68e5dcfb9413f53afd9c8f86e56a7b4d86d9a2fa26090ea2dc9e40fba56c6ec6" [[package]] name = "windows_aarch64_msvc" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" +checksum = "8dab469ebbc45798319e69eebf92308e541ce46760b49b18c6b3fe5e8965b30f" [[package]] name = "windows_i686_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4e9b6a7cac734a8b4138a4e1044eac3404d8326b6c0f939276560687a033fb" [[package]] name = "windows_i686_msvc" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" +checksum = "28b0ec9c422ca95ff34a78755cfa6ad4a51371da2a5ace67500cf7ca5f232c58" [[package]] name = "windows_x86_64_gnu" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "704131571ba93e89d7cd43482277d6632589b18ecf4468f591fbae0a8b101614" [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.0" +version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42079295511643151e98d61c38c0acc444e52dd42ab456f7ccfd5152e8ecf21c" [[package]] name = "windows_x86_64_msvc" -version = "0.48.0" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" +checksum = "0770833d60a970638e989b3fa9fd2bb1aaadcf88963d1659fd7d9990196ed2d6" diff --git a/src/tests/libpq_test/Cargo.toml b/src/tests/libpq_test/Cargo.toml index 813cd37ca90b6..846a2e1059d66 100644 --- a/src/tests/libpq_test/Cargo.toml +++ b/src/tests/libpq_test/Cargo.toml @@ -10,6 +10,3 @@ edition = "2021" anyhow = "1" libpq = "3.0" clap = { version = "4", features = ["derive"] } - -[lints] -workspace = true diff --git a/src/tests/regress/data/expected/contrib-pgcrypto-rijndael.out b/src/tests/regress/data/expected/contrib-pgcrypto-rijndael.out new file mode 100644 index 0000000000000..015ba4430d962 --- /dev/null +++ b/src/tests/regress/data/expected/contrib-pgcrypto-rijndael.out @@ -0,0 +1,137 @@ +-- +-- AES cipher (aka Rijndael-128, -192, or -256) +-- +-- some standard Rijndael testvalues +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f', +'aes-ecb/pad:none'); + encrypt +------------------------------------ + \x69c4e0d86a7b0430d8cdb78070b4c55a +(1 row) + +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f1011121314151617', +'aes-ecb/pad:none'); + encrypt +------------------------------------ + \xdda97ca4864cdfe06eaf70a0ec0d7191 +(1 row) + +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f', +'aes-ecb/pad:none'); + encrypt +------------------------------------ + \x8ea2b7ca516745bfeafc49904b496089 +(1 row) + +-- cbc +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f', +'aes-cbc/pad:none'); + encrypt +------------------------------------ + \x8ea2b7ca516745bfeafc49904b496089 +(1 row) + +-- without padding, input not multiple of block size +SELECT encrypt( +'\x00112233445566778899aabbccddeeff00', +'\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f', +'aes-cbc/pad:none'); +ERROR: encrypt error: Encryption failed +-- key padding +SELECT encrypt( +'\x0011223344', +'\x000102030405', +'aes-cbc'); + encrypt +------------------------------------ + \x189a28932213f017b246678dbc28655f +(1 row) + +SELECT encrypt( +'\x0011223344', +'\x000102030405060708090a0b0c0d0e0f10111213', +'aes-cbc'); + encrypt +------------------------------------ + \x3b02279162d15580e069d3a71407a556 +(1 row) + +SELECT encrypt( +'\x0011223344', +'\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b', +'aes-cbc'); + encrypt +------------------------------------ + \x4facb6a041d53e0a5a73289170901fe7 +(1 row) + +-- empty data +select encrypt('', 'foo', 'aes'); + encrypt +------------------------------------ + \xb48cc3338a2eb293b6007ef72c360d48 +(1 row) + +-- 10 bytes key +select encrypt('foo', '0123456789', 'aes'); + encrypt +------------------------------------ + \xf397f03d2819b7172b68d0706fda4693 +(1 row) + +-- 22 bytes key +select encrypt('foo', '0123456789012345678901', 'aes'); + encrypt +------------------------------------ + \x5c9db77af02b4678117bcd8a71ae7f53 +(1 row) + +-- decrypt +select encode(decrypt(encrypt('foo', '0123456', 'aes'), '0123456', 'aes'), 'escape'); + encode +-------- + foo +(1 row) + +-- data not multiple of block size +select encode(decrypt(encrypt('foo', '0123456', 'aes') || '\x00'::bytea, '0123456', 'aes'), 'escape'); +ERROR: decrypt error: Decryption failed +-- bad padding +-- (The input value is the result of encrypt_iv('abcdefghijklmnopqrstuvwxyz', '0123456', 'abcd', 'aes') +-- with the 16th byte changed (s/db/eb/) to corrupt the padding of the last block.) +select encode(decrypt_iv('\xa21a9c15231465964e3396d32095e67eb52bab05f556a581621dee1b85385789', '0123456', 'abcd', 'aes'), 'escape'); +ERROR: decrypt_iv error: Decryption failed +-- iv +select encrypt_iv('foo', '0123456', 'abcd', 'aes'); + encrypt_iv +------------------------------------ + \x2c24cb7da91d6d5699801268b0f5adad +(1 row) + +select encode(decrypt_iv('\x2c24cb7da91d6d5699801268b0f5adad', '0123456', 'abcd', 'aes'), 'escape'); + encode +-------- + foo +(1 row) + +-- long message +select encrypt('Lets try a longer message.', '0123456789', 'aes'); + encrypt +-------------------------------------------------------------------- + \xd9beb785dd5403ed02f66b755bb191b93ed93ca54930153f2c3b9ec7785056ad +(1 row) + +select encode(decrypt(encrypt('Lets try a longer message.', '0123456789', 'aes'), '0123456789', 'aes'), 'escape'); + encode +---------------------------- + Lets try a longer message. +(1 row) + diff --git a/src/tests/regress/data/schedule b/src/tests/regress/data/schedule index 044e6b40516f1..d1cfd9ab86253 100644 --- a/src/tests/regress/data/schedule +++ b/src/tests/regress/data/schedule @@ -12,3 +12,4 @@ test: strings date time timestamp interval test: case arrays delete test: jsonb jsonb_jsonpath test: regex +test: contrib-pgcrypto-rijndael diff --git a/src/tests/regress/data/sql/contrib-pgcrypto-rijndael.sql b/src/tests/regress/data/sql/contrib-pgcrypto-rijndael.sql new file mode 100644 index 0000000000000..3bcc47494fe67 --- /dev/null +++ b/src/tests/regress/data/sql/contrib-pgcrypto-rijndael.sql @@ -0,0 +1,72 @@ +-- +-- AES cipher (aka Rijndael-128, -192, or -256) +-- + +-- some standard Rijndael testvalues +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f', +'aes-ecb/pad:none'); + +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f1011121314151617', +'aes-ecb/pad:none'); + +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f', +'aes-ecb/pad:none'); + +-- cbc +SELECT encrypt( +'\x00112233445566778899aabbccddeeff', +'\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f', +'aes-cbc/pad:none'); + +-- without padding, input not multiple of block size +SELECT encrypt( +'\x00112233445566778899aabbccddeeff00', +'\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f', +'aes-cbc/pad:none'); + +-- key padding + +--@ SELECT encrypt( +--@ '\x0011223344', +--@ '\x000102030405', +--@ 'aes-cbc'); +--@ +--@ SELECT encrypt( +--@ '\x0011223344', +--@ '\x000102030405060708090a0b0c0d0e0f10111213', +--@ 'aes-cbc'); +--@ +--@ SELECT encrypt( +--@ '\x0011223344', +--@ '\x000102030405060708090a0b0c0d0e0f101112131415161718191a1b', +--@ 'aes-cbc'); +--@ +--@ -- empty data +--@ select encrypt('', 'foo', 'aes'); +--@ -- 10 bytes key +--@ select encrypt('foo', '0123456789', 'aes'); +--@ -- 22 bytes key +--@ select encrypt('foo', '0123456789012345678901', 'aes'); +--@ +--@ -- decrypt +--@ select encode(decrypt(encrypt('foo', '0123456', 'aes'), '0123456', 'aes'), 'escape'); +-- data not multiple of block size +select encode(decrypt(encrypt('foo', '0123456', 'aes') || '\x00'::bytea, '0123456', 'aes'), 'escape'); +-- bad padding +-- (The input value is the result of encrypt_iv('abcdefghijklmnopqrstuvwxyz', '0123456', 'abcd', 'aes') +-- with the 16th byte changed (s/db/eb/) to corrupt the padding of the last block.) +select encode(decrypt_iv('\xa21a9c15231465964e3396d32095e67eb52bab05f556a581621dee1b85385789', '0123456', 'abcd', 'aes'), 'escape'); + +-- iv +--@ select encrypt_iv('foo', '0123456', 'abcd', 'aes'); +--@ select encode(decrypt_iv('\x2c24cb7da91d6d5699801268b0f5adad', '0123456', 'abcd', 'aes'), 'escape'); + +-- long message +--@ select encrypt('Lets try a longer message.', '0123456789', 'aes'); +--@ select encode(decrypt(encrypt('Lets try a longer message.', '0123456789', 'aes'), '0123456789', 'aes'), 'escape'); diff --git a/src/tests/regress/src/lib.rs b/src/tests/regress/src/lib.rs index efdd0f1422c00..a18afdebc843f 100644 --- a/src/tests/regress/src/lib.rs +++ b/src/tests/regress/src/lib.rs @@ -27,6 +27,9 @@ #![deny(rustdoc::broken_intra_doc_links)] #![feature(path_file_prefix)] #![feature(let_chains)] +#![feature(register_tool)] +#![register_tool(rw)] +#![allow(rw::format_error)] mod opts; diff --git a/src/tests/simulation/Cargo.toml b/src/tests/simulation/Cargo.toml index 4911b7ceb67cc..2acca6a4c012a 100644 --- a/src/tests/simulation/Cargo.toml +++ b/src/tests/simulation/Cargo.toml @@ -23,7 +23,7 @@ fail = { version = "0.5" } futures = { version = "0.3", default-features = false, features = ["alloc"] } glob = "0.3" itertools = "0.12" -lru = { git = "https://github.com/risingwavelabs/lru-rs.git", rev = "cb2d7c7" } +lru = { workspace = true } madsim = "0.2.22" paste = "1" pin-project = "1.1" @@ -42,6 +42,7 @@ risingwave_expr_impl = { workspace = true } risingwave_frontend = { workspace = true } risingwave_hummock_sdk = { workspace = true } risingwave_meta_node = { workspace = true } +risingwave_object_store = { workspace = true } risingwave_pb = { workspace = true } risingwave_rpc_client = { workspace = true } risingwave_sqlparser = { workspace = true } diff --git a/src/tests/simulation/src/cluster.rs b/src/tests/simulation/src/cluster.rs index 9b3e1a7ad7369..f3d2f0559e998 100644 --- a/src/tests/simulation/src/cluster.rs +++ b/src/tests/simulation/src/cluster.rs @@ -32,6 +32,8 @@ use itertools::Itertools; use madsim::runtime::{Handle, NodeHandle}; use rand::seq::IteratorRandom; use rand::Rng; +#[cfg(madsim)] +use risingwave_object_store::object::sim::SimServer as ObjectStoreSimServer; use risingwave_pb::common::WorkerNode; use sqllogictest::AsyncDB; #[cfg(not(madsim))] @@ -94,8 +96,23 @@ pub struct Configuration { impl Default for Configuration { fn default() -> Self { + let config_path = { + let mut file = + tempfile::NamedTempFile::new().expect("failed to create temp config file"); + + let config_data = r#" +[server] +telemetry_enabled = false +metrics_level = "Disabled" +"# + .to_string(); + file.write_all(config_data.as_bytes()) + .expect("failed to write config file"); + file.into_temp_path() + }; + Configuration { - config_path: ConfigPath::Regular("".into()), + config_path: ConfigPath::Temp(config_path.into()), frontend_nodes: 1, compute_nodes: 1, meta_nodes: 1, @@ -109,7 +126,7 @@ impl Default for Configuration { } impl Configuration { - /// Returns the config for scale test. + /// Returns the configuration for scale test. pub fn for_scale() -> Self { // Embed the config file and create a temporary file at runtime. The file will be deleted // automatically when it's dropped. @@ -128,7 +145,31 @@ impl Configuration { meta_nodes: 3, compactor_nodes: 2, compute_node_cores: 2, - per_session_queries: vec!["SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=false".into()] + ..Default::default() + } + } + + /// Provides a configuration for scale test which ensures that the arrangement backfill is disabled, + /// so table scan will use `no_shuffle`. + pub fn for_scale_no_shuffle() -> Self { + // Embed the config file and create a temporary file at runtime. The file will be deleted + // automatically when it's dropped. + let config_path = { + let mut file = + tempfile::NamedTempFile::new().expect("failed to create temp config file"); + file.write_all(include_bytes!("risingwave-scale.toml")) + .expect("failed to write config file"); + file.into_temp_path() + }; + + Configuration { + config_path: ConfigPath::Temp(config_path.into()), + frontend_nodes: 2, + compute_nodes: 3, + meta_nodes: 3, + compactor_nodes: 2, + compute_node_cores: 2, + per_session_queries: vec!["SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL = false;".into()] .into(), ..Default::default() } @@ -136,9 +177,10 @@ impl Configuration { pub fn for_auto_parallelism( max_heartbeat_interval_secs: u64, - enable_auto_scale_in: bool, enable_auto_parallelism: bool, ) -> Self { + let disable_automatic_parallelism_control = !enable_auto_parallelism; + let config_path = { let mut file = tempfile::NamedTempFile::new().expect("failed to create temp config file"); @@ -146,8 +188,10 @@ impl Configuration { let config_data = format!( r#"[meta] max_heartbeat_interval_secs = {max_heartbeat_interval_secs} -enable_scale_in_when_recovery = {enable_auto_scale_in} -enable_automatic_parallelism_control = {enable_auto_parallelism} +disable_automatic_parallelism_control = {disable_automatic_parallelism_control} +parallelism_control_trigger_first_delay_sec = 0 +parallelism_control_batch_size = 0 +parallelism_control_trigger_period_sec = 10 [system] barrier_interval_ms = 250 @@ -170,7 +214,10 @@ metrics_level = "Disabled" meta_nodes: 1, compactor_nodes: 1, compute_node_cores: 2, - per_session_queries: vec!["SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL=false".into()] + per_session_queries: vec![ + "create view if not exists table_parallelism as select t.name, tf.parallelism from rw_tables t, rw_table_fragments tf where t.id = tf.table_id;".into(), + "create view if not exists mview_parallelism as select m.name, tf.parallelism from rw_materialized_views m, rw_table_fragments tf where m.id = tf.table_id;".into(), + ] .into(), ..Default::default() } @@ -217,6 +264,8 @@ metrics_level = "Disabled" meta_nodes: 1, compactor_nodes: 1, compute_node_cores: 1, + per_session_queries: vec!["SET STREAMING_ENABLE_ARRANGEMENT_BACKFILL = true;".into()] + .into(), ..Default::default() } } @@ -253,18 +302,18 @@ metrics_level = "Disabled" /// /// # Nodes /// -/// | Name | IP | -/// | -------------- | ------------- | -/// | meta-x | 192.168.1.x | -/// | frontend-x | 192.168.2.x | -/// | compute-x | 192.168.3.x | -/// | compactor-x | 192.168.4.x | -/// | etcd | 192.168.10.1 | -/// | kafka-broker | 192.168.11.1 | -/// | kafka-producer | 192.168.11.2 | -/// | s3 | 192.168.12.1 | -/// | client | 192.168.100.1 | -/// | ctl | 192.168.101.1 | +/// | Name | IP | +/// | ---------------- | ------------- | +/// | meta-x | 192.168.1.x | +/// | frontend-x | 192.168.2.x | +/// | compute-x | 192.168.3.x | +/// | compactor-x | 192.168.4.x | +/// | etcd | 192.168.10.1 | +/// | kafka-broker | 192.168.11.1 | +/// | kafka-producer | 192.168.11.2 | +/// | object_store_sim | 192.168.12.1 | +/// | client | 192.168.100.1 | +/// | ctl | 192.168.101.1 | pub struct Cluster { config: Configuration, handle: Handle, @@ -341,14 +390,13 @@ impl Cluster { }) .build(); - // s3 + // object_store_sim handle .create_node() - .name("s3") + .name("object_store_sim") .ip("192.168.12.1".parse().unwrap()) .init(move || async move { - aws_sdk_s3::server::SimServer::default() - .with_bucket("hummock001") + ObjectStoreSimServer::builder() .serve("0.0.0.0:9301".parse().unwrap()) .await }) @@ -378,7 +426,7 @@ impl Cluster { "--etcd-endpoints", "etcd:2388", "--state-store", - "hummock+minio://hummockadmin:hummockadmin@192.168.12.1:9301/hummock001", + "hummock+sim://hummockadmin:hummockadmin@192.168.12.1:9301/hummock001", "--data-directory", "hummock_001", ]); @@ -819,6 +867,13 @@ impl Session { self.run("FLUSH").await?; Ok(()) } + + pub async fn is_arrangement_backfill_enabled(&mut self) -> Result { + let result = self + .run("show streaming_enable_arrangement_backfill") + .await?; + Ok(result == "true") + } } /// Options for killing nodes. diff --git a/src/tests/simulation/src/ctl_ext.rs b/src/tests/simulation/src/ctl_ext.rs index 5a880ec05dfd9..77ae0ee414e49 100644 --- a/src/tests/simulation/src/ctl_ext.rs +++ b/src/tests/simulation/src/ctl_ext.rs @@ -15,6 +15,7 @@ #![cfg_attr(not(madsim), expect(unused_imports))] use std::collections::{HashMap, HashSet}; +use std::ffi::OsString; use std::fmt::Write; use std::sync::Arc; @@ -380,7 +381,6 @@ impl Cluster { .spawn(async move { let revision = format!("{}", revision); let mut v = vec![ - "ctl", "meta", "reschedule", "--plan", @@ -393,8 +393,7 @@ impl Cluster { v.push("--resolve-no-shuffle"); } - let opts = risingwave_ctl::CliOpts::parse_from(v); - risingwave_ctl::start(opts).await + start_ctl(v).await }) .await??; @@ -404,26 +403,14 @@ impl Cluster { /// Pause all data sources in the cluster. #[cfg_or_panic(madsim)] pub async fn pause(&mut self) -> Result<()> { - self.ctl - .spawn(async move { - let opts = risingwave_ctl::CliOpts::parse_from(["ctl", "meta", "pause"]); - risingwave_ctl::start(opts).await - }) - .await??; - + self.ctl.spawn(start_ctl(["meta", "pause"])).await??; Ok(()) } /// Resume all data sources in the cluster. #[cfg_or_panic(madsim)] pub async fn resume(&mut self) -> Result<()> { - self.ctl - .spawn(async move { - let opts = risingwave_ctl::CliOpts::parse_from(["ctl", "meta", "resume"]); - risingwave_ctl::start(opts).await - }) - .await??; - + self.ctl.spawn(start_ctl(["meta", "resume"])).await??; Ok(()) } @@ -433,7 +420,6 @@ impl Cluster { self.ctl .spawn(async move { let mut command: Vec = vec![ - "ctl".into(), "throttle".into(), "mv".into(), table_id.table_id.to_string(), @@ -441,8 +427,7 @@ impl Cluster { if let Some(rate_limit) = rate_limit { command.push(rate_limit.to_string()); } - let opts = risingwave_ctl::CliOpts::parse_from(command); - risingwave_ctl::start(opts).await + start_ctl(command).await }) .await??; Ok(()) @@ -479,3 +464,14 @@ impl Cluster { Ok(resp) } } + +#[cfg_attr(not(madsim), allow(dead_code))] +async fn start_ctl(args: I) -> Result<()> +where + S: Into, + I: IntoIterator, +{ + let args = std::iter::once("ctl".into()).chain(args.into_iter().map(|s| s.into())); + let opts = risingwave_ctl::CliOpts::parse_from(args); + risingwave_ctl::start_fallible(opts).await +} diff --git a/src/tests/simulation/src/main.rs b/src/tests/simulation/src/main.rs index 9de1fc6d79a30..cf976095328ba 100644 --- a/src/tests/simulation/src/main.rs +++ b/src/tests/simulation/src/main.rs @@ -144,6 +144,10 @@ pub struct Args { /// The probability of background ddl for a ddl query. #[clap(long, default_value = "0.0")] background_ddl_rate: f64, + + /// Use arrangement backfill by default + #[clap(long, default_value = "false")] + use_arrangement_backfill: bool, } #[tokio::main] @@ -173,6 +177,11 @@ async fn main() { meta_nodes: args.meta_nodes, etcd_timeout_rate: args.etcd_timeout_rate, etcd_data_path: args.etcd_data, + per_session_queries: if args.use_arrangement_backfill { + vec!["SET enable_arrangement_backfill = true;".to_string()].into() + } else { + vec![].into() + }, ..Default::default() }; let kill_opts = KillOpts { diff --git a/src/tests/simulation/tests/integration_tests/backfill_tests.rs b/src/tests/simulation/tests/integration_tests/backfill_tests.rs index 168e84a7ff5a1..18b8e7303f02d 100644 --- a/src/tests/simulation/tests/integration_tests/backfill_tests.rs +++ b/src/tests/simulation/tests/integration_tests/backfill_tests.rs @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::thread::sleep; use std::time::Duration; use anyhow::Result; use itertools::Itertools; use risingwave_simulation::cluster::{Cluster, Configuration}; +use tokio::time::{sleep, timeout}; + +use crate::utils::kill_cn_and_wait_recover; const SET_PARALLELISM: &str = "SET STREAMING_PARALLELISM=1;"; const ROOT_TABLE_CREATE: &str = "create table t1 (_id int, data jsonb);"; @@ -150,7 +152,7 @@ async fn test_arrangement_backfill_replication() -> Result<()> { let upstream_task = tokio::spawn(async move { // The initial 100 records will take approx 3s // After that we start ingesting upstream records. - sleep(Duration::from_secs(3)); + sleep(Duration::from_secs(3)).await; for i in 101..=200 { session2 .run(format!("insert into t values ({})", i)) @@ -190,7 +192,96 @@ async fn test_arrangement_backfill_replication() -> Result<()> { Ok(()) } +#[tokio::test] +async fn test_backfill_backpressure() -> Result<()> { + let mut cluster = Cluster::start(Configuration::default()).await?; + let mut session = cluster.start_session(); + + // Create dimension table + session.run("CREATE TABLE dim (v1 int);").await?; + // Ingest + // Amplification of 200 records + session + .run("INSERT INTO dim SELECT 1 FROM generate_series(1, 200);") + .await?; + // Create fact table + session.run("CREATE TABLE fact (v1 int);").await?; + // Create sink + session + .run("CREATE SINK s1 AS SELECT fact.v1 FROM fact JOIN dim ON fact.v1 = dim.v1 with (connector='blackhole');") + .await?; + session.run("FLUSH").await?; + + // Ingest + tokio::spawn(async move { + session + .run("INSERT INTO fact SELECT 1 FROM generate_series(1, 100000);") + .await + .unwrap(); + }) + .await?; + let mut session = cluster.start_session(); + session.run("FLUSH").await?; + // Run flush to check if barrier can go through. It should be able to. + // There will be some latency for the initial barrier. + session.run("FLUSH;").await?; + // But after that flush should be processed timely. + timeout(Duration::from_secs(1), session.run("FLUSH;")).await??; + Ok(()) +} + // TODO(kwannoel): Test case where upstream distribution is Single, then downstream // distribution MUST also be single, and arrangement backfill should just use Simple. // TODO(kwannoel): Test arrangement backfill background recovery. +#[tokio::test] +async fn test_arrangement_backfill_progress() -> Result<()> { + let mut cluster = Cluster::start(Configuration::for_arrangement_backfill()).await?; + let mut session = cluster.start_session(); + + // Create base table + session.run("CREATE TABLE t (v1 int primary key)").await?; + + // Ingest data + session + .run("INSERT INTO t SELECT * FROM generate_series(1, 1000)") + .await?; + session.run("FLUSH;").await?; + + // Create arrangement backfill with rate limit + session.run("SET STREAMING_PARALLELISM=1").await?; + session.run("SET BACKGROUND_DDL=true").await?; + session.run("SET STREAMING_RATE_LIMIT=1").await?; + session + .run("CREATE MATERIALIZED VIEW m1 AS SELECT * FROM t") + .await?; + + // Verify arrangement backfill progress after 10s, it should be 1% at least. + sleep(Duration::from_secs(10)).await; + let progress = session + .run("SELECT progress FROM rw_catalog.rw_ddl_progress") + .await?; + let progress = progress.replace('%', ""); + let progress = progress.parse::().unwrap(); + assert!( + (1.0..2.0).contains(&progress), + "progress not within bounds {}", + progress + ); + + // Trigger recovery and test it again. + kill_cn_and_wait_recover(&cluster).await; + let prev_progress = progress; + let progress = session + .run("SELECT progress FROM rw_catalog.rw_ddl_progress") + .await?; + let progress = progress.replace('%', ""); + let progress = progress.parse::().unwrap(); + assert!( + (prev_progress - 0.5..prev_progress + 1.5).contains(&progress), + "progress not within bounds {}", + progress + ); + + Ok(()) +} diff --git a/src/tests/simulation/tests/integration_tests/main.rs b/src/tests/simulation/tests/integration_tests/main.rs index 43262cd7b52a2..475793a88b709 100644 --- a/src/tests/simulation/tests/integration_tests/main.rs +++ b/src/tests/simulation/tests/integration_tests/main.rs @@ -28,3 +28,5 @@ mod scale; mod sink; mod storage; mod throttle; + +mod utils; diff --git a/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs index 44e574f66a690..4c6f743f08f8b 100644 --- a/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs +++ b/src/tests/simulation/tests/integration_tests/recovery/background_ddl.rs @@ -14,11 +14,15 @@ use std::time::Duration; -use anyhow::Result; -use risingwave_common::error::anyhow_error; -use risingwave_simulation::cluster::{Cluster, Configuration, KillOpts, Session}; +use anyhow::{anyhow, Result}; +use risingwave_common::error::v2::AsReport; +use risingwave_simulation::cluster::{Cluster, Configuration, Session}; use tokio::time::sleep; +use crate::utils::{ + kill_cn_and_meta_and_wait_recover, kill_cn_and_wait_recover, kill_random_and_wait_recover, +}; + const CREATE_TABLE: &str = "CREATE TABLE t(v1 int);"; const DROP_TABLE: &str = "DROP TABLE t;"; const SEED_TABLE_500: &str = "INSERT INTO t SELECT generate_series FROM generate_series(1, 500);"; @@ -31,39 +35,6 @@ const CREATE_MV1: &str = "CREATE MATERIALIZED VIEW mv1 as SELECT * FROM t;"; const DROP_MV1: &str = "DROP MATERIALIZED VIEW mv1;"; const WAIT: &str = "WAIT;"; -async fn kill_cn_and_wait_recover(cluster: &Cluster) { - cluster - .kill_nodes(["compute-1", "compute-2", "compute-3"], 0) - .await; - sleep(Duration::from_secs(10)).await; -} - -async fn kill_cn_and_meta_and_wait_recover(cluster: &Cluster) { - cluster - .kill_nodes( - [ - "compute-1", - "compute-2", - "compute-3", - "meta-1", - "meta-2", - "meta-3", - ], - 0, - ) - .await; - sleep(Duration::from_secs(10)).await; -} - -async fn kill_random_and_wait_recover(cluster: &Cluster) { - // Kill it again - for _ in 0..3 { - sleep(Duration::from_secs(2)).await; - cluster.kill_node(&KillOpts::ALL_FAST).await; - } - sleep(Duration::from_secs(10)).await; -} - async fn cancel_stream_jobs(session: &mut Session) -> Result> { tracing::info!("finding streaming jobs to cancel"); let ids = session @@ -78,7 +49,7 @@ async fn cancel_stream_jobs(session: &mut Session) -> Result> { .split('\n') .map(|s| { s.parse::() - .map_err(|_e| anyhow_error!("failed to parse {}", s)) + .map_err(|_e| anyhow!("failed to parse {}", s)) }) .collect::>>()?; Ok(ids) @@ -278,41 +249,44 @@ async fn test_ddl_cancel() -> Result<()> { Ok(()) } -#[tokio::test] -async fn test_high_barrier_latency_cancel() -> Result<()> { +/// When cancelling a stream job under high latency, +/// the cancel should take a long time to take effect. +/// If we trigger a recovery however, the cancel should take effect immediately, +/// since cancel will immediately drop the table fragment. +async fn test_high_barrier_latency_cancel(config: Configuration) -> Result<()> { init_logger(); - let mut cluster = Cluster::start(Configuration::for_scale()).await?; + let mut cluster = Cluster::start(config).await?; let mut session = cluster.start_session(); - // 100,000 fact records - session.run("CREATE TABLE fact (v1 int)").await?; + // Join 2 fact tables together to create a high barrier latency scenario. + + session.run("CREATE TABLE fact1 (v1 int)").await?; session - .run("INSERT INTO fact select 1 from generate_series(1, 100000)") + .run("INSERT INTO fact1 select 1 from generate_series(1, 100000)") .await?; - // Amplification factor of 1000 per record. - session.run("CREATE TABLE dimension (v1 int)").await?; + session.run("CREATE TABLE fact2 (v1 int)").await?; session - .run("INSERT INTO dimension select 1 from generate_series(1, 1000)") + .run("INSERT INTO fact2 select 1 from generate_series(1, 100000)") .await?; session.flush().await?; - // With 10 rate limit, and amplification factor of 1000, - // We should expect 10,000 rows / s. - // That should be enough to cause barrier latency to spike. - session.run("SET STREAMING_RATE_LIMIT=10").await?; - tracing::info!("seeded base tables"); // Create high barrier latency scenario // Keep creating mv1, if it's not created. loop { session.run(SET_BACKGROUND_DDL).await?; - session.run("CREATE MATERIALIZED VIEW mv1 as select fact.v1 from fact join dimension on fact.v1 = dimension.v1").await?; + session.run("CREATE MATERIALIZED VIEW mv1 as select fact1.v1 from fact1 join fact2 on fact1.v1 = fact2.v1").await?; tracing::info!("created mv in background"); sleep(Duration::from_secs(1)).await; - kill_cn_and_wait_recover(&cluster).await; + cluster + .kill_nodes_and_restart(["compute-1", "compute-2", "compute-3"], 2) + .await; + sleep(Duration::from_secs(2)).await; + + tracing::debug!("killed cn, waiting recovery"); // Check if mv stream job is created in the background match session @@ -324,12 +298,7 @@ async fn test_high_barrier_latency_cancel() -> Result<()> { continue; } Err(e) => { - if e.to_string().contains("in creating procedure") { - // MV already created and recovered. - break; - } else { - return Err(e); - } + return Err(e); } Ok(s) => { tracing::info!("created mv stream job with status: {}", s); @@ -340,35 +309,71 @@ async fn test_high_barrier_latency_cancel() -> Result<()> { tracing::info!("restarted cn: trigger stream job recovery"); - // Attempt to cancel - let mut session2 = cluster.start_session(); - let handle = tokio::spawn(async move { - let result = cancel_stream_jobs(&mut session2).await; - assert!(result.is_err()) - }); - - sleep(Duration::from_secs(2)).await; - kill_cn_and_wait_recover(&cluster).await; - tracing::info!("restarted cn: cancel should take effect"); + // Make sure there's some progress first. + loop { + // Wait until at least 10% of records are ingested. + let progress = session + .run("select progress from rw_catalog.rw_ddl_progress;") + .await + .unwrap(); + tracing::info!(progress, "get progress before cancel stream job"); + let progress = progress.replace('%', ""); + let progress = progress.parse::().unwrap(); + if progress > 0.01 { + break; + } else { + sleep(Duration::from_micros(1)).await; + } + } + // Loop in case the cancel gets dropped after + // cn kill, before it drops the table fragment. + for iteration in 0..5 { + tracing::info!(iteration, "cancelling stream job"); + let mut session2 = cluster.start_session(); + let handle = tokio::spawn(async move { + let result = cancel_stream_jobs(&mut session2).await; + tracing::info!(?result, "cancel stream jobs"); + }); + + sleep(Duration::from_millis(500)).await; + kill_cn_and_wait_recover(&cluster).await; + tracing::info!("restarted cn: cancel should take effect"); - handle.await.unwrap(); + handle.await.unwrap(); - // Create MV with same relation name should succeed, - // since the previous job should be cancelled. - tracing::info!("recreating mv"); - session.run("SET BACKGROUND_DDL=false").await?; - session - .run("CREATE MATERIALIZED VIEW mv1 as values(1)") - .await?; - tracing::info!("recreated mv"); + // Create MV with same relation name should succeed, + // since the previous job should be cancelled. + tracing::info!("recreating mv"); + session.run("SET BACKGROUND_DDL=false").await?; + if let Err(e) = session + .run("CREATE MATERIALIZED VIEW mv1 as values(1)") + .await + { + tracing::info!(error = %e.as_report(), "Recreate mv failed"); + continue; + } else { + tracing::info!("recreated mv"); + break; + } + } session.run(DROP_MV1).await?; - session.run("DROP TABLE fact").await?; - session.run("DROP TABLE dimension").await?; + session.run("DROP TABLE fact1").await?; + session.run("DROP TABLE fact2").await?; Ok(()) } +#[tokio::test] +async fn test_high_barrier_latency_cancel_for_arrangement_backfill() -> Result<()> { + test_high_barrier_latency_cancel(Configuration::for_arrangement_backfill()).await +} + +#[tokio::test] +async fn test_high_barrier_latency_cancel_for_no_shuffle() -> Result<()> { + test_high_barrier_latency_cancel(Configuration::for_scale_no_shuffle()).await +} + // When cluster stop, foreground ddl job must be cancelled. #[tokio::test] async fn test_foreground_ddl_no_recovery() -> Result<()> { diff --git a/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs b/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs index 242d699c8259d..4943516bb9ddc 100644 --- a/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs +++ b/src/tests/simulation/tests/integration_tests/scale/auto_parallelism.rs @@ -32,7 +32,6 @@ async fn test_passive_online_and_offline() -> Result<()> { let config = Configuration::for_auto_parallelism( MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE, true, - true, ); let mut cluster = Cluster::start(config.clone()).await?; let mut session = cluster.start_session(); @@ -215,7 +214,6 @@ async fn test_passive_online_and_offline() -> Result<()> { async fn test_active_online() -> Result<()> { let config = Configuration::for_auto_parallelism( MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE, - false, true, ); let mut cluster = Cluster::start(config.clone()).await?; @@ -302,7 +300,6 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper( ) -> Result<()> { let config = Configuration::for_auto_parallelism( MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE, - true, enable_auto_parallelism_control, ); let mut cluster = Cluster::start(config.clone()).await?; @@ -324,7 +321,7 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper( session .run("select parallelism from rw_table_fragments") .await? - .assert_result_eq("AUTO"); + .assert_result_eq("ADAPTIVE"); async fn locate_table_fragment(cluster: &mut Cluster) -> Result { cluster @@ -434,12 +431,14 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper( // We alter parallelism back to auto - session.run("alter table t set parallelism = auto").await?; + session + .run("alter table t set parallelism = adaptive") + .await?; session .run("select parallelism from rw_table_fragments") .await? - .assert_result_eq("AUTO"); + .assert_result_eq("ADAPTIVE"); let table_mat_fragment = locate_table_fragment(&mut cluster).await?; @@ -488,11 +487,13 @@ async fn test_auto_parallelism_control_with_fixed_and_auto_helper( async fn test_compatibility_with_low_level() -> Result<()> { let config = Configuration::for_auto_parallelism( MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE, - false, true, ); let mut cluster = Cluster::start(config.clone()).await?; let mut session = cluster.start_session(); + session + .run("SET streaming_enable_arrangement_backfill = false;") + .await?; // Keep one worker reserved for adding later. let select_worker = "compute-2"; @@ -505,10 +506,6 @@ async fn test_compatibility_with_low_level() -> Result<()> { )) .await; - // helper views - session.run("create view table_parallelism as select t.name, tf.parallelism from rw_tables t, rw_table_fragments tf where t.id = tf.table_id;").await?; - session.run("create view mview_parallelism as select m.name, tf.parallelism from rw_materialized_views m, rw_table_fragments tf where m.id = tf.table_id;").await?; - session.run("create table t(v int);").await?; // single fragment downstream @@ -524,7 +521,7 @@ async fn test_compatibility_with_low_level() -> Result<()> { session .run("select parallelism from table_parallelism") .await? - .assert_result_eq("AUTO"); + .assert_result_eq("ADAPTIVE"); let table_mat_fragment = cluster .locate_one_fragment(vec![ @@ -555,7 +552,7 @@ async fn test_compatibility_with_low_level() -> Result<()> { session .run("select parallelism from mview_parallelism where name = 'm_simple'") .await? - .assert_result_eq("AUTO"); + .assert_result_eq("ADAPTIVE"); let simple_mv_fragment = cluster .locate_one_fragment(vec![ @@ -576,16 +573,16 @@ async fn test_compatibility_with_low_level() -> Result<()> { // Since `m_simple` only has 1 fragment, and this fragment is a downstream of NO_SHUFFLE relation, // in reality, `m_simple` does not have a fragment of its own. // Therefore, any low-level modifications to this fragment will only be passed up to the highest level through the NO_SHUFFLE relationship and then passed back down. - // Hence, the parallelism of `m_simple` should still be equivalent to AUTO of 0 fragment. + // Hence, the parallelism of `m_simple` should still be equivalent to ADAPTIVE of 0 fragment. session .run("select parallelism from mview_parallelism where name = 'm_simple'") .await? - .assert_result_eq("AUTO"); + .assert_result_eq("ADAPTIVE"); session .run("select parallelism from mview_parallelism where name = 'm_join'") .await? - .assert_result_eq("AUTO"); + .assert_result_eq("ADAPTIVE"); let hash_join_fragment = cluster .locate_one_fragment(vec![identity_contains("hashJoin")]) @@ -593,7 +590,7 @@ async fn test_compatibility_with_low_level() -> Result<()> { let hash_join_fragment_id = hash_join_fragment.id(); - // manual scale in m_simple materialize fragment + // manual scale in m_join materialize fragment cluster .reschedule_resolve_no_shuffle(format!( "{hash_join_fragment_id}-[{chosen_parallel_unit_a}]" @@ -626,3 +623,116 @@ async fn test_compatibility_with_low_level() -> Result<()> { Ok(()) } + +#[tokio::test] +async fn test_compatibility_with_low_level_and_arrangement_backfill() -> Result<()> { + let config = Configuration::for_auto_parallelism( + MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE, + true, + ); + let mut cluster = Cluster::start(config.clone()).await?; + let mut session = cluster.start_session(); + + // Keep one worker reserved for adding later. + let select_worker = "compute-2"; + cluster + .simple_kill_nodes(vec![select_worker.to_string()]) + .await; + + sleep(Duration::from_secs( + MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE * 2, + )) + .await; + + session.run("create table t(v int);").await?; + + // Streaming arrangement backfill + session + .run("SET streaming_enable_arrangement_backfill = true;") + .await?; + session + .run("create materialized view m_simple as select * from t;") + .await?; + + session + .run("select parallelism from table_parallelism") + .await? + .assert_result_eq("ADAPTIVE"); + + // Find the table materialize fragment + let table_mat_fragment = cluster + .locate_one_fragment(vec![ + identity_contains("materialize"), + identity_contains("union"), + ]) + .await?; + + let (mut all_parallel_units, _) = table_mat_fragment.parallel_unit_usage(); + + let chosen_parallel_unit_a = all_parallel_units.pop().unwrap(); + let chosen_parallel_unit_b = all_parallel_units.pop().unwrap(); + + let table_mat_fragment_id = table_mat_fragment.id(); + + // manual scale in table materialize fragment + cluster + .reschedule(format!( + "{table_mat_fragment_id}-[{chosen_parallel_unit_a}]", + )) + .await?; + + session + .run("select parallelism from table_parallelism") + .await? + .assert_result_eq("CUSTOM"); + + // Upstream changes should not affect downstream. + session + .run("select parallelism from mview_parallelism where name = 'm_simple'") + .await? + .assert_result_eq("ADAPTIVE"); + + // Find the table fragment for materialized view + let simple_mv_fragment = cluster + .locate_one_fragment(vec![ + identity_contains("materialize"), + identity_contains("StreamTableScan"), + ]) + .await?; + + let simple_mv_fragment_id = simple_mv_fragment.id(); + + // manual scale in m_simple materialize fragment + cluster + .reschedule_resolve_no_shuffle(format!( + "{simple_mv_fragment_id}-[{chosen_parallel_unit_b}]", + )) + .await?; + + // The downstream table fragment should be separate from the upstream table fragment. + session + .run("select parallelism from mview_parallelism where name = 'm_simple'") + .await? + .assert_result_eq("CUSTOM"); + + let before_fragment_parallelism = session + .run("select fragment_id, parallelism from rw_fragments order by fragment_id;") + .await?; + + cluster + .simple_restart_nodes(vec![select_worker.to_string()]) + .await; + + sleep(Duration::from_secs( + MAX_HEARTBEAT_INTERVAL_SECS_CONFIG_FOR_AUTO_SCALE * 2, + )) + .await; + + let after_fragment_parallelism = session + .run("select fragment_id, parallelism from rw_fragments order by fragment_id;") + .await?; + + assert_eq!(before_fragment_parallelism, after_fragment_parallelism); + + Ok(()) +} diff --git a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs index 6e1c1333007d3..981f79103403d 100644 --- a/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs +++ b/src/tests/simulation/tests/integration_tests/scale/cascade_materialized_view.rs @@ -32,6 +32,7 @@ const MV5: &str = "create materialized view m5 as select * from m4;"; async fn test_simple_cascade_materialized_view() -> Result<()> { let mut cluster = Cluster::start(Configuration::for_scale()).await?; let mut session = cluster.start_session(); + let arrangement_backfill_is_enabled = session.is_arrangement_backfill_enabled().await?; session.run(ROOT_TABLE_CREATE).await?; session.run(MV1).await?; @@ -57,10 +58,19 @@ async fn test_simple_cascade_materialized_view() -> Result<()> { .locate_one_fragment([identity_contains("StreamTableScan")]) .await?; - assert_eq!( - chain_fragment.inner.actors.len(), - fragment.inner.actors.len() - ); + if arrangement_backfill_is_enabled { + // The chain fragment is in a different table fragment. + assert_eq!(chain_fragment.inner.actors.len(), 6,); + // The upstream materialized fragment should be scaled in + assert_eq!(fragment.inner.actors.len(), 1,); + } else { + // No shuffle, so the fragment of upstream materialized node is the same + // as stream table scan. + assert_eq!( + chain_fragment.inner.actors.len(), + fragment.inner.actors.len() + ); + } session .run(&format!( diff --git a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs index c269be2c0fa34..a409443371ba9 100644 --- a/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs +++ b/src/tests/simulation/tests/integration_tests/scale/no_shuffle.rs @@ -20,7 +20,7 @@ use risingwave_simulation::utils::AssertResult; #[tokio::test] async fn test_delta_join() -> Result<()> { - let mut cluster = Cluster::start(Configuration::for_scale()).await?; + let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?; let mut session = cluster.start_session(); session.run("set rw_implicit_flush = true;").await?; @@ -125,7 +125,7 @@ async fn test_delta_join() -> Result<()> { #[tokio::test] async fn test_share_multiple_no_shuffle_upstream() -> Result<()> { - let mut cluster = Cluster::start(Configuration::for_scale()).await?; + let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?; let mut session = cluster.start_session(); session.run("create table t (a int, b int);").await?; @@ -145,7 +145,7 @@ async fn test_share_multiple_no_shuffle_upstream() -> Result<()> { #[tokio::test] async fn test_resolve_no_shuffle_upstream() -> Result<()> { - let mut cluster = Cluster::start(Configuration::for_scale()).await?; + let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?; let mut session = cluster.start_session(); session.run("create table t (v int);").await?; diff --git a/src/tests/simulation/tests/integration_tests/scale/plan.rs b/src/tests/simulation/tests/integration_tests/scale/plan.rs index 24b294ad39c6f..e1d8148632c62 100644 --- a/src/tests/simulation/tests/integration_tests/scale/plan.rs +++ b/src/tests/simulation/tests/integration_tests/scale/plan.rs @@ -175,7 +175,7 @@ async fn test_resize_single() -> Result<()> { #[tokio::test] async fn test_resize_single_failed() -> Result<()> { - let mut cluster = Cluster::start(Configuration::for_scale()).await?; + let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?; let mut session = cluster.start_session(); session.run("create table t (v int);").await?; @@ -245,7 +245,7 @@ async fn test_resize_single_failed() -> Result<()> { } #[tokio::test] async fn test_resize_no_shuffle() -> Result<()> { - let mut cluster = Cluster::start(Configuration::for_scale()).await?; + let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?; let mut session = cluster.start_session(); session.run("create table t (v int);").await?; diff --git a/src/tests/simulation/tests/integration_tests/scale/table.rs b/src/tests/simulation/tests/integration_tests/scale/table.rs index 04293c1701bc1..6bceb5c07b536 100644 --- a/src/tests/simulation/tests/integration_tests/scale/table.rs +++ b/src/tests/simulation/tests/integration_tests/scale/table.rs @@ -88,7 +88,7 @@ async fn test_mv_on_scaled_table() -> Result<()> { #[tokio::test] async fn test_scale_on_schema_change() -> Result<()> { - let mut cluster = Cluster::start(Configuration::for_scale()).await?; + let mut cluster = Cluster::start(Configuration::for_scale_no_shuffle()).await?; cluster.run(ROOT_TABLE_CREATE).await?; cluster.run(MV1).await?; diff --git a/src/tests/simulation/tests/integration_tests/sink/basic.rs b/src/tests/simulation/tests/integration_tests/sink/basic.rs index fe9aad07568b8..8ba8982ce4d7a 100644 --- a/src/tests/simulation/tests/integration_tests/sink/basic.rs +++ b/src/tests/simulation/tests/integration_tests/sink/basic.rs @@ -17,6 +17,7 @@ use std::sync::Arc; use std::time::Duration; use anyhow::Result; +use itertools::Itertools; use tokio::time::sleep; use crate::sink::utils::{ @@ -45,11 +46,54 @@ async fn basic_test_inner(is_decouple: bool) -> Result<()> { session.run(CREATE_SINK).await?; assert_eq!(6, test_sink.parallelism_counter.load(Relaxed)); + let internal_tables = session.run("show internal tables").await?; + + let table_name_prefix = "__internal_test_sink_"; + + let sink_internal_table_name: String = TryInto::<[&str; 1]>::try_into( + internal_tables + .split("\n") + .filter(|line| { + line.contains(table_name_prefix) + && line + .strip_prefix(table_name_prefix) + .unwrap() + .contains("sink") + }) + .collect_vec(), + ) + .unwrap()[0] + .to_string(); + + let _result = session + .run(format!( + "select * from {} limit 10", + sink_internal_table_name + )) + .await?; + + let _result = session + .run(format!( + "select * from {} where kv_log_store_vnode = 0 limit 10", + sink_internal_table_name + )) + .await?; + test_sink .store .wait_for_count(test_source.id_list.len()) .await?; + let result: String = session.run("select * from rw_sink_decouple").await?; + let [_, is_sink_decouple_str, vnode_count_str] = + TryInto::<[&str; 3]>::try_into(result.split(" ").collect_vec()).unwrap(); + if is_decouple { + assert_eq!(is_sink_decouple_str, "t"); + assert_eq!(vnode_count_str, "256"); + } else { + assert_eq!(is_sink_decouple_str, "f"); + } + session.run(DROP_SINK).await?; session.run(DROP_SOURCE).await?; diff --git a/src/tests/simulation/tests/integration_tests/sink/utils.rs b/src/tests/simulation/tests/integration_tests/sink/utils.rs index 6b9ea61e708b1..a134ab1a265ff 100644 --- a/src/tests/simulation/tests/integration_tests/sink/utils.rs +++ b/src/tests/simulation/tests/integration_tests/sink/utils.rs @@ -38,7 +38,6 @@ use risingwave_connector::sink::SinkError; use risingwave_connector::source::test_source::{ registry_test_source, BoxSource, TestSourceRegistryGuard, TestSourceSplit, }; -use risingwave_connector::source::StreamChunkWithState; use risingwave_simulation::cluster::{Cluster, ConfigPath, Configuration}; use tokio::time::sleep; @@ -230,20 +229,30 @@ impl SimulationTestSink { } } -pub fn build_stream_chunk(row_iter: impl Iterator) -> StreamChunk { +pub fn build_stream_chunk( + row_iter: impl Iterator, +) -> StreamChunk { static ROW_ID_GEN: LazyLock> = LazyLock::new(|| Arc::new(AtomicI64::new(0))); let mut builder = DataChunkBuilder::new( - vec![DataType::Int32, DataType::Varchar, DataType::Serial], + vec![ + DataType::Int32, + DataType::Varchar, + DataType::Serial, + DataType::Varchar, + DataType::Varchar, + ], 100000, ); - for (id, name) in row_iter { + for (id, name, split_id, offset) in row_iter { let row_id = ROW_ID_GEN.fetch_add(1, Relaxed); std::assert!(builder .append_one_row([ Some(ScalarImpl::Int32(id)), Some(ScalarImpl::Utf8(name.into())), Some(ScalarImpl::Serial(Serial::from(row_id))), + Some(ScalarImpl::Utf8(split_id.into())), + Some(ScalarImpl::Utf8(offset.into())), ]) .is_none()); } @@ -297,20 +306,19 @@ impl SimulationTestSource { split.offset.parse::().unwrap() + 1 }; - let mut stream: BoxStream<'static, StreamChunkWithState> = empty().boxed(); + let mut stream: BoxStream<'static, StreamChunk> = empty().boxed(); while offset < id_list.len() { let mut chunks = Vec::new(); while offset < id_list.len() && chunks.len() < pause_interval { let id = id_list[offset]; - let chunk = build_stream_chunk(once((id, simple_name_of_id(id)))); - let mut split_offset = HashMap::new(); - split_offset.insert(split.id.clone(), offset.to_string()); - let chunk_with_state = StreamChunkWithState { - chunk, - split_offset_mapping: Some(split_offset), - }; - chunks.push(chunk_with_state); + let chunk = build_stream_chunk(once(( + id, + simple_name_of_id(id), + split.id.to_string(), + offset.to_string(), + ))); + chunks.push(chunk); offset += 1; } @@ -332,7 +340,7 @@ impl SimulationTestSource { } stream - .chain(async { pending::().await }.into_stream()) + .chain(async { pending::().await }.into_stream()) .map(|chunk| Ok(chunk)) .boxed() })) diff --git a/src/tests/simulation/tests/integration_tests/utils.rs b/src/tests/simulation/tests/integration_tests/utils.rs new file mode 100644 index 0000000000000..8f06d0acbea2f --- /dev/null +++ b/src/tests/simulation/tests/integration_tests/utils.rs @@ -0,0 +1,51 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::time::Duration; + +use risingwave_simulation::cluster::{Cluster, KillOpts}; +use tokio::time::sleep; + +pub(crate) async fn kill_cn_and_wait_recover(cluster: &Cluster) { + cluster + .kill_nodes(["compute-1", "compute-2", "compute-3"], 0) + .await; + sleep(Duration::from_secs(10)).await; +} + +pub(crate) async fn kill_cn_and_meta_and_wait_recover(cluster: &Cluster) { + cluster + .kill_nodes( + [ + "compute-1", + "compute-2", + "compute-3", + "meta-1", + "meta-2", + "meta-3", + ], + 0, + ) + .await; + sleep(Duration::from_secs(10)).await; +} + +pub(crate) async fn kill_random_and_wait_recover(cluster: &Cluster) { + // Kill it again + for _ in 0..3 { + sleep(Duration::from_secs(2)).await; + cluster.kill_node(&KillOpts::ALL_FAST).await; + } + sleep(Duration::from_secs(10)).await; +} diff --git a/src/tests/sqlsmith/Cargo.toml b/src/tests/sqlsmith/Cargo.toml index a83bb60b2b8f3..8e76dc20d71b8 100644 --- a/src/tests/sqlsmith/Cargo.toml +++ b/src/tests/sqlsmith/Cargo.toml @@ -28,6 +28,7 @@ risingwave_frontend = { workspace = true } risingwave_pb = { workspace = true } risingwave_sqlparser = { workspace = true } similar = "2.4.0" +thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio" } tokio-postgres = "0.7" tracing = "0.1" @@ -38,7 +39,7 @@ workspace-hack = { path = "../../workspace-hack" } [dev-dependencies] expect-test = "1" -libtest-mimic = "0.6" +libtest-mimic = "0.7" [[bin]] name = "sqlsmith" diff --git a/src/tests/sqlsmith/src/lib.rs b/src/tests/sqlsmith/src/lib.rs index 2d8c23a52b740..34ab90cb497a6 100644 --- a/src/tests/sqlsmith/src/lib.rs +++ b/src/tests/sqlsmith/src/lib.rs @@ -274,6 +274,7 @@ CREATE TABLE t3(v1 int, v2 bool, v3 smallint); options: [], }, ], + wildcard_idx: None, constraints: [], with_options: [], source_schema: None, @@ -319,6 +320,7 @@ CREATE TABLE t3(v1 int, v2 bool, v3 smallint); options: [], }, ], + wildcard_idx: None, constraints: [], with_options: [], source_schema: None, @@ -375,6 +377,7 @@ CREATE TABLE t3(v1 int, v2 bool, v3 smallint); options: [], }, ], + wildcard_idx: None, constraints: [], with_options: [], source_schema: None, @@ -507,6 +510,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY ], }, ], + wildcard_idx: None, constraints: [], with_options: [], source_schema: None, @@ -559,6 +563,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY ], }, ], + wildcard_idx: None, constraints: [], with_options: [], source_schema: None, @@ -618,6 +623,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY ], }, ], + wildcard_idx: None, constraints: [], with_options: [], source_schema: None, @@ -695,6 +701,7 @@ CREATE TABLE t4(v1 int PRIMARY KEY, v2 smallint PRIMARY KEY, v3 bool PRIMARY KEY ], }, ], + wildcard_idx: None, constraints: [], with_options: [], source_schema: None, diff --git a/src/tests/sqlsmith/src/sql_gen/relation.rs b/src/tests/sqlsmith/src/sql_gen/relation.rs index 05d67d96c4222..a5c6b7d27545d 100644 --- a/src/tests/sqlsmith/src/sql_gen/relation.rs +++ b/src/tests/sqlsmith/src/sql_gen/relation.rs @@ -219,11 +219,8 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { right_table: Table, ) -> Option { // We always generate an equi join, to avoid stream nested loop join. - let Some((base_join_on_expr, remaining_equi_columns)) = - self.gen_single_equi_join_expr(left_columns, right_columns) - else { - return None; - }; + let (base_join_on_expr, remaining_equi_columns) = + self.gen_single_equi_join_expr(left_columns, right_columns)?; // Add more expressions let extra_expr = match self.rng.gen_range(1..=100) { @@ -263,11 +260,8 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { right_columns: Vec, right_table: Table, ) -> Option { - let Some(join_constraint) = - self.gen_join_constraint(left_columns, left_table, right_columns, right_table) - else { - return None; - }; + let join_constraint = + self.gen_join_constraint(left_columns, left_table, right_columns, right_table)?; // NOTE: INNER JOIN works fine, usually does not encounter `StreamNestedLoopJoin` much. // If many failures due to `StreamNestedLoopJoin`, try disable the others. @@ -290,14 +284,12 @@ impl<'a, R: Rng> SqlGenerator<'a, R> { let left_columns = left_table.get_qualified_columns(); let (right_factor, right_table) = self.gen_table_factor(); let right_columns = right_table.get_qualified_columns(); - let Some(join_operator) = self.gen_join_operator( + let join_operator = self.gen_join_operator( left_columns, left_table.clone(), right_columns, right_table.clone(), - ) else { - return None; - }; + )?; let right_factor_with_join = Join { relation: right_factor, diff --git a/src/tests/sqlsmith/src/sql_gen/types.rs b/src/tests/sqlsmith/src/sql_gen/types.rs index b713689218a67..b4f9d01d294a9 100644 --- a/src/tests/sqlsmith/src/sql_gen/types.rs +++ b/src/tests/sqlsmith/src/sql_gen/types.rs @@ -267,9 +267,7 @@ pub(crate) static BINARY_INEQUALITY_OP_TABLE: LazyLock< .filter_map(|func| { let lhs = func.inputs_type[0].as_exact().clone(); let rhs = func.inputs_type[1].as_exact().clone(); - let Some(op) = expr_type_to_inequality_op(func.name.as_scalar()) else { - return None; - }; + let op = expr_type_to_inequality_op(func.name.as_scalar())?; Some(((lhs, rhs), op)) }) .for_each(|(args, op)| funcs.entry(args).or_default().push(op)); diff --git a/src/tests/sqlsmith/tests/frontend/mod.rs b/src/tests/sqlsmith/tests/frontend/mod.rs index 8cb869aaf0e6f..5c58f028b933d 100644 --- a/src/tests/sqlsmith/tests/frontend/mod.rs +++ b/src/tests/sqlsmith/tests/frontend/mod.rs @@ -28,6 +28,7 @@ use risingwave_sqlparser::ast::Statement; use risingwave_sqlsmith::{ is_permissible_error, mview_sql_gen, parse_create_table_statements, parse_sql, sql_gen, Table, }; +use thiserror_ext::AsReport; use tokio::runtime::Runtime; type Result = std::result::Result; @@ -48,7 +49,7 @@ async fn handle(session: Arc, stmt: Statement, sql: Arc) -> Re let result = handler::handle(session.clone(), stmt, sql, vec![]) .await .map(|_| ()) - .map_err(|e| format!("Error Reason:\n{}", e).into()); + .map_err(|e| format!("Error Reason:\n{}", e.as_report()).into()); validate_result(result) } @@ -183,20 +184,26 @@ fn run_batch_query( let mut binder = Binder::new(&session); let bound = binder .bind(stmt) - .map_err(|e| Failed::from(format!("Failed to bind:\nReason:\n{}", e)))?; + .map_err(|e| Failed::from(format!("Failed to bind:\nReason:\n{}", e.as_report())))?; let mut planner = Planner::new(context); - let mut logical_plan = planner - .plan(bound) - .map_err(|e| Failed::from(format!("Failed to generate logical plan:\nReason:\n{}", e)))?; - let batch_plan = logical_plan - .gen_batch_plan() - .map_err(|e| Failed::from(format!("Failed to generate batch plan:\nReason:\n{}", e)))?; + let mut logical_plan = planner.plan(bound).map_err(|e| { + Failed::from(format!( + "Failed to generate logical plan:\nReason:\n{}", + e.as_report() + )) + })?; + let batch_plan = logical_plan.gen_batch_plan().map_err(|e| { + Failed::from(format!( + "Failed to generate batch plan:\nReason:\n{}", + e.as_report() + )) + })?; logical_plan .gen_batch_distributed_plan(batch_plan) .map_err(|e| { Failed::from(format!( "Failed to generate batch distributed plan:\nReason:\n{}", - e + e.as_report() )) })?; Ok(()) diff --git a/src/utils/delta_btree_map/Cargo.toml b/src/utils/delta_btree_map/Cargo.toml index 8ae7245b2741d..48534ffad1867 100644 --- a/src/utils/delta_btree_map/Cargo.toml +++ b/src/utils/delta_btree_map/Cargo.toml @@ -15,6 +15,7 @@ ignored = ["workspace-hack"] normal = ["workspace-hack"] [dependencies] +educe = "0.5" enum-as-inner = "0.6" [lints] diff --git a/src/utils/delta_btree_map/src/lib.rs b/src/utils/delta_btree_map/src/lib.rs index 120b02d9d0b2d..c300955af567b 100644 --- a/src/utils/delta_btree_map/src/lib.rs +++ b/src/utils/delta_btree_map/src/lib.rs @@ -18,11 +18,13 @@ use std::cmp::Ordering; use std::collections::BTreeMap; use std::ops::Bound; +use educe::Educe; use enum_as_inner::EnumAsInner; /// [`DeltaBTreeMap`] wraps two [`BTreeMap`] references respectively as snapshot and delta, /// providing cursor that can iterate over the updated version of the snapshot. -#[derive(Debug, Clone, Copy)] +#[derive(Debug, Educe)] +#[educe(Clone, Copy)] pub struct DeltaBTreeMap<'a, K: Ord, V> { snapshot: &'a BTreeMap, delta: &'a BTreeMap>, diff --git a/src/utils/futures_util/Cargo.toml b/src/utils/futures_util/Cargo.toml new file mode 100644 index 0000000000000..97bd794daaf8d --- /dev/null +++ b/src/utils/futures_util/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "rw_futures_util" +edition = { workspace = true } +license = { workspace = true } +repository = { workspace = true } +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[package.metadata.cargo-machete] +ignored = ["workspace-hack"] + +[package.metadata.cargo-udeps.ignore] +normal = ["workspace-hack"] + +[dependencies] +futures = "0.3" +pin-project-lite = "0.2" + +[dev-dependencies] +tokio = { version = "1", features = ["full"] } + +[lints] +workspace = true diff --git a/src/common/src/util/future_utils/buffered_with_fence.rs b/src/utils/futures_util/src/buffered_with_fence.rs similarity index 80% rename from src/common/src/util/future_utils/buffered_with_fence.rs rename to src/utils/futures_util/src/buffered_with_fence.rs index 30b1938fda991..6271fd189a587 100644 --- a/src/common/src/util/future_utils/buffered_with_fence.rs +++ b/src/utils/futures_util/src/buffered_with_fence.rs @@ -45,6 +45,21 @@ pin_project! { } } +impl TryBufferedWithFence +where + St: TryStream, + St::Ok: TryFuture + MaybeFence, +{ + pub(crate) fn new(stream: St, n: usize) -> Self { + Self { + stream: stream.into_stream().fuse(), + in_progress_queue: FuturesOrdered::new(), + syncing: false, + max: n, + } + } +} + impl Stream for TryBufferedWithFence where St: TryStream, @@ -100,6 +115,15 @@ pin_project! { } } +impl Fenced +where + Fut: Future, +{ + pub(crate) fn new(inner: Fut, is_fence: bool) -> Self { + Self { inner, is_fence } + } +} + impl Future for Fenced where Fut: Future, @@ -131,51 +155,6 @@ where } } -pub trait RwFutureExt: Future { - fn with_fence(self, is_fence: bool) -> Fenced - where - Self: Sized; -} - -impl RwFutureExt for Fut { - fn with_fence(self, is_fence: bool) -> Fenced { - Fenced { - inner: self, - is_fence, - } - } -} - -pub trait RwTryStreamExt: TryStream { - /// Similar to [`TryStreamExt::try_buffered`](https://docs.rs/futures/latest/futures/stream/trait.TryStreamExt.html#method.try_buffered), but respect to fence. - /// - /// Fence is provided by [`Future`] that implements [`MaybeFence`] and returns `true`. - /// When the stream receive a fenced future, it'll not do a sync operation. In brief, don't poll later futures until the current - /// buffer is cleared. - fn try_buffered_with_fence(self, n: usize) -> TryBufferedWithFence - where - Self: Sized, - Self::Ok: TryFuture + MaybeFence; -} - -impl RwTryStreamExt for St -where - St: TryStream, -{ - fn try_buffered_with_fence(self, n: usize) -> TryBufferedWithFence - where - Self: Sized, - Self::Ok: TryFuture + MaybeFence, - { - TryBufferedWithFence { - stream: self.into_stream().fuse(), - in_progress_queue: FuturesOrdered::new(), - syncing: false, - max: n, - } - } -} - #[cfg(test)] mod tests { use std::sync::{Arc, Mutex}; @@ -183,7 +162,7 @@ mod tests { use futures::stream::StreamExt; - use super::{RwFutureExt, RwTryStreamExt}; + use crate::{RwFutureExt, RwTryStreamExt}; #[tokio::test] async fn test_buffered_with_fence() { diff --git a/src/utils/futures_util/src/lib.rs b/src/utils/futures_util/src/lib.rs new file mode 100644 index 0000000000000..4d086951dbb5f --- /dev/null +++ b/src/utils/futures_util/src/lib.rs @@ -0,0 +1,73 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![feature(lint_reasons)] + +use std::future::Future; + +use futures::stream::TryStream; +use futures::{Stream, TryFuture}; + +mod buffered_with_fence; +mod misc; +mod pausable; + +use buffered_with_fence::{Fenced, MaybeFence, TryBufferedWithFence}; +pub use misc::*; +pub use pausable::{Pausable, Valve}; + +/// Create a pausable stream, which can be paused or resumed by a valve. +pub fn pausable(stream: St) -> (Pausable, Valve) +where + St: Stream, +{ + Pausable::new(stream) +} + +pub trait RwTryStreamExt: TryStream { + /// Similar to [`TryStreamExt::try_buffered`](https://docs.rs/futures/latest/futures/stream/trait.TryStreamExt.html#method.try_buffered), but respect to fence. + /// + /// Fence is provided by [`Future`] that implements [`MaybeFence`] and returns `true`. + /// When the stream receive a fenced future, it'll not do a sync operation. In brief, don't poll later futures until the current + /// buffer is cleared. + fn try_buffered_with_fence(self, n: usize) -> TryBufferedWithFence + where + Self: Sized, + Self::Ok: TryFuture + MaybeFence; +} + +impl RwTryStreamExt for St +where + St: TryStream, +{ + fn try_buffered_with_fence(self, n: usize) -> TryBufferedWithFence + where + Self: Sized, + Self::Ok: TryFuture + MaybeFence, + { + TryBufferedWithFence::new(self, n) + } +} + +pub trait RwFutureExt: Future { + fn with_fence(self, is_fence: bool) -> Fenced + where + Self: Sized; +} + +impl RwFutureExt for Fut { + fn with_fence(self, is_fence: bool) -> Fenced { + Fenced::new(self, is_fence) + } +} diff --git a/src/common/src/util/future_utils/mod.rs b/src/utils/futures_util/src/misc.rs similarity index 67% rename from src/common/src/util/future_utils/mod.rs rename to src/utils/futures_util/src/misc.rs index d71ebfa7d6765..f224b58214ddf 100644 --- a/src/common/src/util/future_utils/mod.rs +++ b/src/utils/futures_util/src/misc.rs @@ -12,15 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -mod buffered_with_fence; - -use std::future::pending; +use std::future::Future; use std::pin::{pin, Pin}; +use std::task::{ready, Context, Poll}; -pub use buffered_with_fence::*; -use futures::future::{select, Either}; +use futures::future::{pending, select, Either}; use futures::stream::Peekable; -use futures::{Future, FutureExt, Stream, StreamExt}; +use futures::{FutureExt, Stream, StreamExt}; /// Convert a list of streams into a [`Stream`] of results from the streams. pub fn select_all( @@ -79,3 +77,45 @@ pub async fn await_future_with_monitor_error_stream( Either::Right((output, _)) => Ok(output), } } + +/// Attach an item of type `T` to the future `F`. When the future is polled with ready, +/// the item will be attached to the output of future as `(F::Output, item)`. +/// +/// The generated future will be similar to `future.map(|output| (output, item))`. The +/// only difference is that the `Map` future does not provide method `into_inner` to +/// get the original inner future. +pub struct AttachedFuture { + inner: F, + item: Option, +} + +impl AttachedFuture { + pub fn new(inner: F, item: T) -> Self { + Self { + inner, + item: Some(item), + } + } + + pub fn into_inner(self) -> (F, T) { + ( + self.inner, + self.item.expect("should not be called after polled ready"), + ) + } +} + +impl Future for AttachedFuture { + type Output = (F::Output, T); + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + let this = self.get_mut(); + let output = ready!(this.inner.poll_unpin(cx)); + Poll::Ready(( + output, + this.item + .take() + .expect("should not be polled ready for twice"), + )) + } +} diff --git a/src/utils/futures_util/src/pausable.rs b/src/utils/futures_util/src/pausable.rs new file mode 100644 index 0000000000000..531c832413d48 --- /dev/null +++ b/src/utils/futures_util/src/pausable.rs @@ -0,0 +1,96 @@ +// Copyright 2024 RisingWave Labs +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll, Waker}; + +use futures::Stream; +use pin_project_lite::pin_project; + +pin_project! { + #[derive(Debug)] + #[must_use = "streams do nothing unless polled"] + pub struct Pausable + where St: Stream + { + #[pin] + stream: St, + paused: Arc, + waker: Arc>>, + } +} + +/// A valve is a handle that can control the [`Pausable`] stream. +#[derive(Clone)] +pub struct Valve { + paused: Arc, + waker: Arc>>, +} + +impl Valve { + /// Pause the stream controlled by the valve. + pub fn pause(&self) { + self.paused.store(true, Ordering::Relaxed); + } + + /// Resume the stream controlled by the valve. + pub fn resume(&self) { + self.paused.store(false, Ordering::Relaxed); + if let Some(waker) = self.waker.lock().unwrap().as_ref() { + waker.wake_by_ref() + } + } +} + +impl Pausable +where + St: Stream, +{ + pub(crate) fn new(stream: St) -> (Self, Valve) { + let paused = Arc::new(AtomicBool::new(false)); + let waker = Arc::new(Mutex::new(None)); + ( + Pausable { + stream, + paused: paused.clone(), + waker: waker.clone(), + }, + Valve { paused, waker }, + ) + } +} + +impl Stream for Pausable +where + St: Stream, +{ + type Item = St::Item; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.project(); + if this.paused.load(Ordering::Relaxed) { + let mut waker = this.waker.lock().unwrap(); + *waker = Some(cx.waker().clone()); + Poll::Pending + } else { + this.stream.poll_next(cx) + } + } + + fn size_hint(&self) -> (usize, Option) { + self.stream.size_hint() + } +} diff --git a/src/utils/pgwire/Cargo.toml b/src/utils/pgwire/Cargo.toml index 51588721fb7de..47840b0cf4983 100644 --- a/src/utils/pgwire/Cargo.toml +++ b/src/utils/pgwire/Cargo.toml @@ -21,10 +21,15 @@ byteorder = "1.5" bytes = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } itertools = "0.12" +jsonwebtoken = "9" openssl = "0.10.60" panic-message = "0.3" +parking_lot = "0.12" +reqwest = { version = "0.11" } risingwave_common = { workspace = true } risingwave_sqlparser = { workspace = true } +serde = { version = "1", features = ["derive"] } +serde_json = "1" thiserror = "1" thiserror-ext = { workspace = true } tokio = { version = "0.2", package = "madsim-tokio", features = ["rt", "macros"] } diff --git a/src/utils/pgwire/src/error.rs b/src/utils/pgwire/src/error.rs index a78ecc7ef1eae..9352c913ef59a 100644 --- a/src/utils/pgwire/src/error.rs +++ b/src/utils/pgwire/src/error.rs @@ -51,6 +51,9 @@ This is a bug. We would appreciate a bug report at: #[error("Unable to setup an SSL connection")] SslError(#[from] openssl::ssl::Error), + + #[error("terminating connection due to idle-in-transaction timeout")] + IdleInTxnTimeout, } impl PsqlError { diff --git a/src/utils/pgwire/src/pg_message.rs b/src/utils/pgwire/src/pg_message.rs index e6211c9a6569b..c69fd6ca5b374 100644 --- a/src/utils/pgwire/src/pg_message.rs +++ b/src/utils/pgwire/src/pg_message.rs @@ -16,6 +16,7 @@ use std::collections::HashMap; use std::ffi::CStr; use std::io::{Error, ErrorKind, IoSlice, Result, Write}; +use anyhow::anyhow; use byteorder::{BigEndian, ByteOrder, NetworkEndian}; /// Part of code learned from . use bytes::{Buf, BufMut, Bytes, BytesMut}; @@ -58,7 +59,7 @@ impl FeStartupMessage { Ok(v) => Ok(v.trim_end_matches('\0')), Err(err) => Err(Error::new( ErrorKind::InvalidInput, - format!("Input end error: {}", err), + anyhow!(err).context("Input end error"), )), }?; let mut map = HashMap::new(); @@ -242,12 +243,12 @@ impl FeQueryMessage { Ok(cstr) => cstr.to_str().map_err(|err| { Error::new( ErrorKind::InvalidInput, - format!("Invalid UTF-8 sequence: {}", err), + anyhow!(err).context("Invalid UTF-8 sequence"), ) }), Err(err) => Err(Error::new( ErrorKind::InvalidInput, - format!("Input end error: {}", err), + anyhow!(err).context("Input end error"), )), } } diff --git a/src/utils/pgwire/src/pg_protocol.rs b/src/utils/pgwire/src/pg_protocol.rs index 2f7c3572ee80a..18411b1a02359 100644 --- a/src/utils/pgwire/src/pg_protocol.rs +++ b/src/utils/pgwire/src/pg_protocol.rs @@ -347,15 +347,16 @@ where self.ready_for_query().ok()?; } - PsqlError::Panic(_) => { + PsqlError::IdleInTxnTimeout | PsqlError::Panic(_) => { self.stream .write_no_flush(&BeMessage::ErrorResponse(Box::new(e))) .ok()?; let _ = self.stream.flush().await; - // Catching the panic during message processing may leave the session in an + // 1. Catching the panic during message processing may leave the session in an // inconsistent state. We forcefully close the connection (then end the // session) here for safety. + // 2. Idle in transaction timeout should also close the connection. return None; } @@ -386,7 +387,7 @@ where match msg { FeMessage::Ssl => self.process_ssl_msg().await?, FeMessage::Startup(msg) => self.process_startup_msg(msg)?, - FeMessage::Password(msg) => self.process_password_msg(msg)?, + FeMessage::Password(msg) => self.process_password_msg(msg).await?, FeMessage::Query(query_msg) => self.process_query_msg(query_msg.get_sql()).await?, FeMessage::CancelQuery(m) => self.process_cancel_msg(m)?, FeMessage::Terminate => self.process_terminate(), @@ -507,7 +508,7 @@ where })?; self.ready_for_query()?; } - UserAuthenticator::ClearText(_) => { + UserAuthenticator::ClearText(_) | UserAuthenticator::OAuth(_) => { self.stream .write_no_flush(&BeMessage::AuthenticationCleartextPassword)?; } @@ -522,11 +523,9 @@ where Ok(()) } - fn process_password_msg(&mut self, msg: FePasswordMessage) -> PsqlResult<()> { + async fn process_password_msg(&mut self, msg: FePasswordMessage) -> PsqlResult<()> { let authenticator = self.session.as_ref().unwrap().user_authenticator(); - if !authenticator.authenticate(&msg.password) { - return Err(PsqlError::PasswordError); - } + authenticator.authenticate(&msg.password).await?; self.stream.write_no_flush(&BeMessage::AuthenticationOk)?; self.stream .write_parameter_status_msg_no_flush(&ParameterStatus::default())?; @@ -550,6 +549,7 @@ where record_sql_in_span(&sql); let session = self.session.clone().unwrap(); + session.check_idle_in_transaction_timeout()?; let _exec_context_guard = session.init_exec_context(sql.clone()); self.inner_process_query_msg(sql.clone(), session.clone()) .await @@ -562,7 +562,9 @@ where ) -> PsqlResult<()> { // Parse sql. let stmts = Parser::parse_sql(&sql) - .inspect_err(|e| tracing::error!("failed to parse sql:\n{}:\n{}", sql, e)) + .inspect_err( + |e| tracing::error!(sql = &*sql, error = %e.as_report(), "failed to parse sql"), + ) .map_err(|err| PsqlError::SimpleQueryError(err.into()))?; if stmts.is_empty() { self.stream.write_no_flush(&BeMessage::EmptyQueryResponse)?; @@ -585,6 +587,7 @@ where session: Arc, ) -> PsqlResult<()> { let session = session.clone(); + // execute query let res = session .clone() @@ -681,7 +684,9 @@ where let stmt = { let stmts = Parser::parse_sql(sql) - .inspect_err(|e| tracing::error!("failed to parse sql:\n{}:\n{}", sql, e)) + .inspect_err( + |e| tracing::error!(sql, error = %e.as_report(), "failed to parse sql"), + ) .map_err(|err| PsqlError::ExtendedPrepareError(err.into()))?; if stmts.len() > 1 { @@ -792,6 +797,7 @@ where let sql: Arc = Arc::from(format!("{}", portal)); record_sql_in_span(&sql); + session.check_idle_in_transaction_timeout()?; let _exec_context_guard = session.init_exec_context(sql.clone()); let result = session.clone().execute(portal).await; @@ -1035,7 +1041,7 @@ where let ssl = openssl::ssl::Ssl::new(ssl_ctx).unwrap(); let mut stream = tokio_openssl::SslStream::new(ssl, stream).unwrap(); if let Err(e) = Pin::new(&mut stream).accept().await { - tracing::warn!("Unable to set up an ssl connection, reason: {}", e); + tracing::warn!(error = %e.as_report(), "Unable to set up an ssl connection"); let _ = stream.shutdown().await; return Err(e.into()); } @@ -1077,7 +1083,7 @@ where Conn::Unencrypted(s) => s.write_no_flush(message), Conn::Ssl(s) => s.write_no_flush(message), } - .inspect_err(|error| tracing::error!(%error, "flush error")) + .inspect_err(|error| tracing::error!(error = %error.as_report(), "flush error")) } async fn write(&mut self, message: &BeMessage<'_>) -> io::Result<()> { @@ -1092,7 +1098,7 @@ where Conn::Unencrypted(s) => s.flush().await, Conn::Ssl(s) => s.flush().await, } - .inspect_err(|error| tracing::error!(%error, "flush error")) + .inspect_err(|error| tracing::error!(error = %error.as_report(), "flush error")) } async fn ssl(&mut self, ssl_ctx: &SslContextRef) -> PsqlResult>> { diff --git a/src/utils/pgwire/src/pg_server.rs b/src/utils/pgwire/src/pg_server.rs index 0a8d1dccddd05..7f6dd41368d45 100644 --- a/src/utils/pgwire/src/pg_server.rs +++ b/src/utils/pgwire/src/pg_server.rs @@ -12,18 +12,24 @@ // See the License for the specific language governing permissions and // limitations under the License. +use std::collections::HashMap; use std::future::Future; use std::io; use std::result::Result; +use std::str::FromStr; use std::sync::Arc; use std::time::Instant; use bytes::Bytes; +use jsonwebtoken::{decode, decode_header, Algorithm, DecodingKey, Validation}; +use parking_lot::Mutex; use risingwave_common::types::DataType; use risingwave_sqlparser::ast::Statement; +use serde::Deserialize; use thiserror_ext::AsReport; use tokio::io::{AsyncRead, AsyncWrite}; +use crate::error::{PsqlError, PsqlResult}; use crate::net::{AddressRef, Listener}; use crate::pg_field_descriptor::PgFieldDescriptor; use crate::pg_message::TransactionStatus; @@ -112,6 +118,8 @@ pub trait Session: Send + Sync { fn transaction_status(&self) -> TransactionStatus; fn init_exec_context(&self, sql: Arc) -> ExecContextGuard; + + fn check_idle_in_transaction_timeout(&self) -> PsqlResult<()>; } /// Each session could run different SQLs multiple times. @@ -120,11 +128,13 @@ pub struct ExecContext { pub running_sql: Arc, /// The instant of the running sql pub last_instant: Instant, + /// A reference used to update when `ExecContext` is dropped + pub last_idle_instant: Arc>>, } /// `ExecContextGuard` holds a `Arc` pointer. Once `ExecContextGuard` is dropped, /// the inner `Arc` should not be referred anymore, so that its `Weak` reference (used in `SessionImpl`) will be the same lifecycle of the running sql execution context. -pub struct ExecContextGuard(Arc); +pub struct ExecContextGuard(#[allow(dead_code)] Arc); impl ExecContextGuard { pub fn new(exec_context: Arc) -> Self { @@ -132,6 +142,12 @@ impl ExecContextGuard { } } +impl Drop for ExecContext { + fn drop(&mut self) { + *self.last_idle_instant.lock() = Some(Instant::now()); + } +} + #[derive(Debug, Clone)] pub enum UserAuthenticator { // No need to authenticate. @@ -143,17 +159,91 @@ pub enum UserAuthenticator { encrypted_password: Vec, salt: [u8; 4], }, + OAuth(HashMap), +} + +/// A JWK Set is a JSON object that represents a set of JWKs. +/// The JSON object MUST have a "keys" member, with its value being an array of JWKs. +/// See for more details. +#[derive(Debug, Deserialize)] +struct Jwks { + keys: Vec, +} + +/// A JSON Web Key (JWK) is a JSON object that represents a cryptographic key. +/// See for more details. +#[derive(Debug, Deserialize)] +struct Jwk { + kid: String, // Key ID + alg: String, // Algorithm + n: String, // Modulus + e: String, // Exponent +} + +async fn validate_jwt( + jwt: &str, + jwks_url: &str, + issuer: &str, + metadata: &HashMap, +) -> Result { + let header = decode_header(jwt)?; + let jwks: Jwks = reqwest::get(jwks_url).await?.json().await?; + + // 1. Retrieve the kid from the header to find the right JWK in the JWK Set. + let kid = header.kid.ok_or("kid not found in jwt header")?; + let jwk = jwks + .keys + .into_iter() + .find(|k| k.kid == kid) + .ok_or("kid not found in jwks")?; + + // 2. Check if the algorithms are matched. + if Algorithm::from_str(&jwk.alg)? != header.alg { + return Err("alg in jwt header does not match with alg in jwk".into()); + } + + // 3. Decode the JWT and validate the claims. + let decoding_key = DecodingKey::from_rsa_components(&jwk.n, &jwk.e)?; + let mut validation = Validation::new(header.alg); + validation.set_issuer(&[issuer]); + validation.set_required_spec_claims(&["exp", "iss"]); + let token_data = decode::>(jwt, &decoding_key, &validation)?; + + // 4. Check if the metadata in the token matches. + if !metadata.iter().all( + |(k, v)| matches!(token_data.claims.get(k), Some(serde_json::Value::String(s)) if s == v), + ) { + return Err("metadata in jwt does not match with metadata declared with user".into()); + } + Ok(true) } impl UserAuthenticator { - pub fn authenticate(&self, password: &[u8]) -> bool { - match self { + pub async fn authenticate(&self, password: &[u8]) -> PsqlResult<()> { + let success = match self { UserAuthenticator::None => true, UserAuthenticator::ClearText(text) => password == text, UserAuthenticator::Md5WithSalt { encrypted_password, .. } => encrypted_password == password, + UserAuthenticator::OAuth(metadata) => { + let mut metadata = metadata.clone(); + let jwks_url = metadata.remove("jwks_url").unwrap(); + let issuer = metadata.remove("issuer").unwrap(); + validate_jwt( + &String::from_utf8_lossy(password), + &jwks_url, + &issuer, + &metadata, + ) + .await + .map_err(PsqlError::StartupError)? + } + }; + if !success { + return Err(PsqlError::PasswordError); } + Ok(()) } } @@ -225,6 +315,7 @@ mod tests { use risingwave_sqlparser::ast::Statement; use tokio_postgres::NoTls; + use crate::error::PsqlResult; use crate::pg_field_descriptor::PgFieldDescriptor; use crate::pg_message::TransactionStatus; use crate::pg_response::{PgResponse, RowSetResult, StatementType}; @@ -362,9 +453,14 @@ mod tests { let exec_context = Arc::new(ExecContext { running_sql: sql, last_instant: Instant::now(), + last_idle_instant: Default::default(), }); ExecContextGuard::new(exec_context) } + + fn check_idle_in_transaction_timeout(&self) -> PsqlResult<()> { + Ok(()) + } } async fn do_test_query(bind_addr: impl Into, pg_config: impl Into) { diff --git a/src/utils/runtime/Cargo.toml b/src/utils/runtime/Cargo.toml index 5d6270e35f6d6..1f4e085632eae 100644 --- a/src/utils/runtime/Cargo.toml +++ b/src/utils/runtime/Cargo.toml @@ -21,8 +21,6 @@ console-subscriber = "0.2.0" either = "1" futures = { version = "0.3", default-features = false, features = ["alloc"] } hostname = "0.3" -opentelemetry-otlp = { version = "0.13" } -opentelemetry-semantic-conventions = "0.12" parking_lot = { version = "0.12", features = ["deadlock_detection"] } pprof = { version = "0.13", features = ["flamegraph"] } risingwave_common = { workspace = true } @@ -40,11 +38,14 @@ tokio = { version = "0.2", package = "madsim-tokio", features = [ "fs" ] } tracing = "0.1" -tracing-opentelemetry = "0.21" +tracing-opentelemetry = { workspace = true } tracing-subscriber = { version = "0.3", features = ["fmt", "parking_lot", "std", "time", "local-time", "json"] } [target.'cfg(not(madsim))'.dependencies] -opentelemetry = { version = "0.20", default-features = false, features = ["rt-tokio"] } +opentelemetry = { workspace = true } +opentelemetry-otlp = { workspace = true } +opentelemetry-semantic-conventions = { workspace = true } +opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] } # only enable `rt-tokio` feature under non-madsim target workspace-hack = { path = "../../workspace-hack" } [lints] diff --git a/src/utils/runtime/src/logger.rs b/src/utils/runtime/src/logger.rs index cb27840d7530f..da77f2e76c0b6 100644 --- a/src/utils/runtime/src/logger.rs +++ b/src/utils/runtime/src/logger.rs @@ -202,8 +202,7 @@ pub fn init_risingwave_logger(settings: LoggerSettings) { // Configure levels for external crates. filter = filter .with_target("foyer", Level::WARN) - .with_target("aws_sdk_ec2", Level::INFO) - .with_target("aws_sdk_s3", Level::INFO) + .with_target("aws", Level::INFO) .with_target("aws_config", Level::WARN) .with_target("aws_endpoint", Level::WARN) .with_target("aws_credential_types::cache::lazy_caching", Level::WARN) @@ -217,6 +216,7 @@ pub fn init_risingwave_logger(settings: LoggerSettings) { .with_target("sled", Level::INFO) .with_target("cranelift", Level::INFO) .with_target("wasmtime", Level::INFO) + .with_target("sqlx", Level::WARN) // Expose hyper connection socket addr log. .with_target("hyper::client::connect::http", Level::DEBUG); @@ -392,8 +392,9 @@ pub fn init_risingwave_logger(settings: LoggerSettings) { if let Some(endpoint) = settings.tracing_endpoint { println!("opentelemetry tracing will be exported to `{endpoint}` if enabled"); - use opentelemetry::{sdk, KeyValue}; + use opentelemetry::KeyValue; use opentelemetry_otlp::WithExportConfig; + use opentelemetry_sdk as sdk; use opentelemetry_semantic_conventions::resource; let id = format!( @@ -435,7 +436,7 @@ pub fn init_risingwave_logger(settings: LoggerSettings) { KeyValue::new(resource::SERVICE_VERSION, env!("CARGO_PKG_VERSION")), KeyValue::new(resource::PROCESS_PID, std::process::id().to_string()), ]))) - .install_batch(opentelemetry::runtime::Tokio) + .install_batch(sdk::runtime::Tokio) .unwrap() }; diff --git a/src/workspace-hack/Cargo.toml b/src/workspace-hack/Cargo.toml index 6b5996fe81d56..75b1555da7ae3 100644 --- a/src/workspace-hack/Cargo.toml +++ b/src/workspace-hack/Cargo.toml @@ -19,6 +19,7 @@ publish = false ### BEGIN HAKARI SECTION [dependencies] ahash = { version = "0.8" } +aho-corasick = { version = "1" } allocator-api2 = { version = "0.2", default-features = false, features = ["alloc", "nightly"] } anyhow = { version = "1", features = ["backtrace"] } async-std = { version = "1", features = ["attributes", "tokio1"] } @@ -29,7 +30,7 @@ aws-sigv4 = { version = "1", features = ["http0-compat", "sign-eventstream", "si aws-smithy-runtime = { version = "1", default-features = false, features = ["client", "rt-tokio", "tls-rustls"] } aws-smithy-types = { version = "1", default-features = false, features = ["byte-stream-poll-next", "http-body-0-4-x", "hyper-0-14-x", "rt-tokio"] } axum = { version = "0.6" } -base64 = { version = "0.21", features = ["alloc"] } +base64 = { version = "0.21" } bigdecimal = { version = "0.4" } bit-vec = { version = "0.6" } bitflags = { version = "2", default-features = false, features = ["serde", "std"] } @@ -46,12 +47,12 @@ deranged = { version = "0.3", default-features = false, features = ["powerfmt", digest = { version = "0.10", features = ["mac", "oid", "std"] } either = { version = "1", features = ["serde"] } fail = { version = "0.5", default-features = false, features = ["failpoints"] } -fixedbitset = { version = "0.4" } flate2 = { version = "1", features = ["zlib"] } frunk_core = { version = "0.4", default-features = false, features = ["std"] } futures = { version = "0.3" } futures-channel = { version = "0.3", features = ["sink"] } futures-core = { version = "0.3" } +futures-executor = { version = "0.3" } futures-io = { version = "0.3" } futures-sink = { version = "0.3" } futures-task = { version = "0.3" } @@ -60,12 +61,11 @@ generic-array = { version = "0.14", default-features = false, features = ["more_ governor = { version = "0.6", default-features = false, features = ["dashmap", "jitter", "std"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["nightly", "raw"] } hashbrown-594e8ee84c453af0 = { package = "hashbrown", version = "0.13", features = ["raw"] } -hashbrown-5ef9efb8ec2df382 = { package = "hashbrown", version = "0.12", features = ["nightly", "raw"] } hmac = { version = "0.12", default-features = false, features = ["reset"] } hyper = { version = "0.14", features = ["full"] } -indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["serde", "std"] } +indexmap-dff4ba8e3ae991db = { package = "indexmap", version = "1", default-features = false, features = ["serde"] } indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] } -itertools = { version = "0.10" } +itertools = { version = "0.11" } jni = { version = "0.21", features = ["invocation"] } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } lexical-core = { version = "0.8", features = ["format"] } @@ -80,21 +80,23 @@ log = { version = "0.4", default-features = false, features = ["kv_unstable", "s madsim-rdkafka = { version = "0.3", features = ["cmake-build", "gssapi", "ssl-vendored", "zstd"] } madsim-tokio = { version = "0.2", default-features = false, features = ["fs", "io-util", "macros", "net", "process", "rt", "rt-multi-thread", "signal", "sync", "time", "tracing"] } md-5 = { version = "0.10" } +memchr = { version = "2" } mio = { version = "0.8", features = ["net", "os-ext"] } +moka = { version = "0.12", features = ["future", "sync"] } nom = { version = "7" } num-bigint = { version = "0.4" } num-integer = { version = "0.1", features = ["i128"] } num-iter = { version = "0.1", default-features = false, features = ["i128", "std"] } num-traits = { version = "0.2", features = ["i128", "libm"] } -opentelemetry_api = { version = "0.20", features = ["logs", "metrics"] } -opentelemetry_sdk = { version = "0.20", features = ["logs", "metrics"] } +openssl = { version = "0.10", features = ["vendored"] } +openssl-sys = { version = "0.9", default-features = false, features = ["vendored"] } ordered-float = { version = "3" } parking_lot = { version = "0.12", features = ["arc_lock", "deadlock_detection"] } parking_lot_core = { version = "0.9", default-features = false, features = ["deadlock_detection"] } petgraph = { version = "0.6" } phf = { version = "0.11", features = ["uncased"] } phf_shared = { version = "0.11", features = ["uncased"] } -postgres-types = { version = "0.2", default-features = false, features = ["derive", "with-chrono-0_4", "with-serde_json-1"] } +postgres-types = { version = "0.2", default-features = false, features = ["derive", "with-chrono-0_4", "with-serde_json-1", "with-uuid-1"] } proc-macro2 = { version = "1", features = ["span-locations"] } prometheus = { version = "0.13", features = ["process"] } prost = { version = "0.12", features = ["no-recursion-limit"] } @@ -130,10 +132,11 @@ sqlx-sqlite = { version = "0.7", default-features = false, features = ["chrono", strum = { version = "0.25", features = ["derive"] } subtle = { version = "2" } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] } +target-lexicon = { version = "0.12", features = ["std"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } tinyvec = { version = "1", features = ["alloc", "grab_spare_slice", "rustc_1_55"] } tokio = { version = "1", features = ["full", "stats", "tracing"] } -tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88", features = ["with-chrono-0_4"] } +tokio-postgres = { git = "https://github.com/madsim-rs/rust-postgres.git", rev = "ac00d88", features = ["with-chrono-0_4", "with-uuid-1"] } tokio-stream = { git = "https://github.com/madsim-rs/tokio.git", rev = "fe39bb8e", features = ["fs", "net"] } tokio-util = { version = "0.7", features = ["codec", "io"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] } @@ -155,6 +158,7 @@ zstd-sys = { version = "2", default-features = false, features = ["legacy", "std [build-dependencies] ahash = { version = "0.8" } +aho-corasick = { version = "1" } allocator-api2 = { version = "0.2", default-features = false, features = ["alloc", "nightly"] } anyhow = { version = "1", features = ["backtrace"] } auto_enums = { version = "0.8", features = ["futures03", "tokio1"] } @@ -164,15 +168,15 @@ cc = { version = "1", default-features = false, features = ["parallel"] } deranged = { version = "0.3", default-features = false, features = ["powerfmt", "serde", "std"] } digest = { version = "0.10", features = ["mac", "oid", "std"] } either = { version = "1", features = ["serde"] } -fixedbitset = { version = "0.4" } frunk_core = { version = "0.4", default-features = false, features = ["std"] } generic-array = { version = "0.14", default-features = false, features = ["more_lengths", "zeroize"] } hashbrown-582f2526e08bb6a0 = { package = "hashbrown", version = "0.14", features = ["nightly", "raw"] } indexmap-f595c2ba2a3f28df = { package = "indexmap", version = "2", features = ["serde"] } -itertools = { version = "0.10" } +itertools = { version = "0.11" } lazy_static = { version = "1", default-features = false, features = ["spin_no_std"] } libc = { version = "0.2", features = ["extra_traits"] } log = { version = "0.4", default-features = false, features = ["kv_unstable", "std"] } +memchr = { version = "2" } nom = { version = "7" } num-bigint = { version = "0.4" } num-integer = { version = "0.1", features = ["i128"] } @@ -198,6 +202,7 @@ sha2 = { version = "0.10", features = ["oid"] } subtle = { version = "2" } syn-dff4ba8e3ae991db = { package = "syn", version = "1", features = ["extra-traits", "full", "visit", "visit-mut"] } syn-f595c2ba2a3f28df = { package = "syn", version = "2", features = ["extra-traits", "fold", "full", "visit", "visit-mut"] } +target-lexicon = { version = "0.12", features = ["std"] } time = { version = "0.3", features = ["local-offset", "macros", "serde-well-known"] } time-macros = { version = "0.2", default-features = false, features = ["formatting", "parsing", "serde"] } toml_datetime = { version = "0.6", default-features = false, features = ["serde"] }